From 1bdf8fa8eeafcf9ed43fa4b11ca90b0e9325f5e6 Mon Sep 17 00:00:00 2001 From: Greg Sjaardema Date: Tue, 7 Jun 2022 07:48:17 -0600 Subject: [PATCH 01/77] Enable compilation with C89 compiler Initializing and declaring variable in for loop statement requires C99 mode. NetCDF is compiled in C89 mode. --- nc_test4/tst_virtual_datasets.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nc_test4/tst_virtual_datasets.c b/nc_test4/tst_virtual_datasets.c index c27455713e..be23deeddb 100644 --- a/nc_test4/tst_virtual_datasets.c +++ b/nc_test4/tst_virtual_datasets.c @@ -12,7 +12,8 @@ static void create_dataset_a() { hsize_t dims[1] = {VARIABLE_SIZE}; float data[VARIABLE_SIZE]; - for(size_t i = 0; i < VARIABLE_SIZE; ++i) { + size_t i; + for(i = 0; i < VARIABLE_SIZE; ++i) { data[i] = (float)i; } @@ -52,6 +53,7 @@ create_dataset_b() { int read_back_contents(const char* filename) { int ncid, varid, ndims; float data[VARIABLE_SIZE]; + size_t i; char var_name[NC_MAX_NAME+1]; int dimids_var[1], var_type, natts; @@ -63,7 +65,7 @@ int read_back_contents(const char* filename) { if (nc_inq_var(ncid, varid, var_name, &var_type, &ndims, dimids_var, &natts)) ERR; if (nc_get_var_float(ncid, varid, data)) ERR; printf("\t %s: ", var_name); - for (size_t i = 0; i < VARIABLE_SIZE; ++i) { + for (i = 0; i < VARIABLE_SIZE; ++i) { printf("%f, ", data[i]); } printf("\n"); From 64d3f2b3263456c922add87633572dccc024cc6c Mon Sep 17 00:00:00 2001 From: wkliao Date: Wed, 20 Apr 2022 00:10:24 -0500 Subject: [PATCH 02/77] count argument in H5Sselect_hyperslab Argument 'count' in NetCDF is not exactly the same as the 'count' in H5Sselect_hyperslabs(space_id, op, start, stride, count, block). When the argument 'stride' is NULL, NetCDF's 'count' should be used in argument 'block', for example, H5Sselect_hyperslabs(space_id, op, start, NULL, ones, count); where 'one' is an array of all 1s. Although using NULL 'block' below H5Sselect_hyperslabs(space_id, op, start, NULL, count, NULL); has the same effect, HDF5 internally stores the space of a subarray as a list of single elements, instead of a "block", which can affect the performance. --- h5_test/tst_h_dimscales2.c | 5 +++-- h5_test/tst_h_enums.c | 4 ++-- h5_test/tst_h_files.c | 10 ++++++---- h5_test/tst_h_par.c | 8 +++++--- h5_test/tst_h_par_compress.c | 8 +++++--- h5_test/tst_h_strings2.c | 7 ++++--- h5_test/tst_h_vars.c | 5 +++-- libhdf5/hdf5var.c | 37 ++++++++++++++++++++++++++++-------- libnczarr/zvar.c | 13 +++++++++---- nc_perf/tst_files3.c | 3 ++- 10 files changed, 68 insertions(+), 32 deletions(-) diff --git a/h5_test/tst_h_dimscales2.c b/h5_test/tst_h_dimscales2.c index 07592e2063..eb7b2dfb53 100644 --- a/h5_test/tst_h_dimscales2.c +++ b/h5_test/tst_h_dimscales2.c @@ -633,6 +633,7 @@ main() hsize_t h5dimlen[DIMS2], h5dimlenmax[DIMS2], xtend_size[DIMS2] = {1, NUM_VALS}; hsize_t start[DIMS2] = {0, 0}; hsize_t count[DIMS2] = {1, NUM_VALS}; + hsize_t ones[DIMS2] = {1, 1}; double value[NUM_VALS]; int dataset_ndims; int i; @@ -661,7 +662,7 @@ main() /* Set up the file and memory spaces. */ if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if ((mem_spaceid = H5Screate_simple(DIMS2, count, NULL)) < 0) ERR; /* Write a slice of data. */ @@ -683,7 +684,7 @@ main() /* Set up the file and memory spaces for a second slice. */ start[0]++; if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if ((mem_spaceid = H5Screate_simple(DIMS2, count, NULL)) < 0) ERR; /* Write a second slice of data. */ diff --git a/h5_test/tst_h_enums.c b/h5_test/tst_h_enums.c index 352b4dce3b..a107023a4f 100644 --- a/h5_test/tst_h_enums.c +++ b/h5_test/tst_h_enums.c @@ -154,7 +154,7 @@ main() char lang[NUM_LANG][STR_LEN + 1] = {"C", "Fortran", "C++", "MISSING"}; enum langs {CLANG=0, Fortran=1, CPP=2, MISSING=255}; short the_value, fill_value = MISSING, data_point = CLANG; - hsize_t start[1] = {1}, count[1] = {1}; + hsize_t start[1] = {1}, count[1] = {1}, one[1] = {1}; int num_members; size_t size; hid_t base_hdf_typeid; @@ -197,7 +197,7 @@ main() if ((mem_spaceid = H5Screate(H5S_SCALAR)) < 0) ERR; if ((file_spaceid = H5Screate_simple(1, dims, NULL)) < 0) ERR; if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, one, count) < 0) ERR; if (H5Dwrite(datasetid, typeid, mem_spaceid, file_spaceid, H5P_DEFAULT, &data_point) < 0) ERR; diff --git a/h5_test/tst_h_files.c b/h5_test/tst_h_files.c index f4bc617f0c..ca795f6e0f 100644 --- a/h5_test/tst_h_files.c +++ b/h5_test/tst_h_files.c @@ -174,7 +174,7 @@ main() #define MILLION 1000000 hid_t fileid, write_spaceid, datasetid, mem_spaceid; - hsize_t start[NDIMS], count[NDIMS]; + hsize_t start[NDIMS], count[NDIMS], ones[NDIMS]; hsize_t dims[1]; int *data; int num_steps; @@ -210,8 +210,9 @@ main() { /* Select hyperslab for write of one slice. */ start[0] = s * SC; + ones[0] = 1; if (H5Sselect_hyperslab(write_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if (H5Dwrite(datasetid, H5T_NATIVE_INT, mem_spaceid, write_spaceid, H5P_DEFAULT, data) < 0) ERR; @@ -242,7 +243,7 @@ main() hid_t mem_spaceid, xfer_plistid, native_typeid; hsize_t *chunksize, dims[1], maxdims[1], *dimsize, *maxdimsize; hsize_t fdims[MAX_DIMS], fmaxdims[MAX_DIMS]; - hsize_t start[MAX_DIMS], count[MAX_DIMS]; + hsize_t start[MAX_DIMS], count[MAX_DIMS], ones[MAX_DIMS]; char file_name[STR_LEN + 1]; char dimscale_wo_var[STR_LEN]; void *bufr; @@ -342,7 +343,8 @@ main() start[1] = 0; count[0] = 1; count[1] = 2097153; - if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) ERR; + ones[0] = ones[1] = 1; + if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL, ones, count) < 0) ERR; if ((mem_spaceid = H5Screate_simple(NDIMS2, count, NULL)) < 0) ERR; if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0) ERR; if ((native_typeid = H5Tget_native_type(H5T_NATIVE_SCHAR, H5T_DIR_DEFAULT)) < 0) ERR; diff --git a/h5_test/tst_h_par.c b/h5_test/tst_h_par.c index 2d350a4537..e8c9c22912 100644 --- a/h5_test/tst_h_par.c +++ b/h5_test/tst_h_par.c @@ -66,7 +66,7 @@ main(int argc, char **argv) printf("*** Creating file for parallel I/O read, and rereading it..."); { hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; - hsize_t start[NDIMS], count[NDIMS]; + hsize_t start[NDIMS], count[NDIMS], ones[NDIMS]; hsize_t dims[1]; int data[SC1], data_in[SC1]; int num_steps; @@ -126,8 +126,9 @@ main(int argc, char **argv) /* Select hyperslab for write of one slice. */ start[0] = s * SC1 * p + my_rank * SC1; count[0] = SC1; + ones[0] = 1; if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid, xferid, data) < 0) ERR; @@ -185,8 +186,9 @@ main(int argc, char **argv) /* Select hyperslab for read of one slice. */ start[0] = s * SC1 * p + my_rank * SC1; count[0] = SC1; + ones[0] = 1; if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) + start, NULL, ones, count) < 0) { ERR; return 2; diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index 6a15e6a810..e634758032 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -51,7 +51,7 @@ main(int argc, char **argv) { hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; hid_t plistid; - hsize_t start[NDIMS], count[NDIMS]; + hsize_t start[NDIMS], count[NDIMS], ones[NDIMS]; hsize_t dims[1], chunksize = SC1; int data[SC1], data_in[SC1]; int num_steps; @@ -120,8 +120,9 @@ main(int argc, char **argv) /* Select hyperslab for write of one slice. */ start[0] = s * SC1 * p + my_rank * SC1; count[0] = SC1; + ones[0] = 1; if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid, xferid, data) < 0) ERR; @@ -160,8 +161,9 @@ main(int argc, char **argv) /* Select hyperslab for read of one slice. */ start[0] = s * SC1 * p + my_rank * SC1; count[0] = SC1; + ones[0] = 1; if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) + start, NULL, ones, count) < 0) { ERR; return 2; diff --git a/h5_test/tst_h_strings2.c b/h5_test/tst_h_strings2.c index 1d03a983c9..a451a6ac64 100644 --- a/h5_test/tst_h_strings2.c +++ b/h5_test/tst_h_strings2.c @@ -30,6 +30,7 @@ main() hid_t file_spaceid, mem_spaceid; hsize_t dims[1] = {0}, max_dims[1] = {H5S_UNLIMITED}, chunk_dims[1] = {1}; hsize_t xtend_size[NDIMS] = {2}, start[NDIMS] = {1}, count[NDIMS] = {1}; + hsize_t ones[NDIMS] = {1}; /* void *fillp;*/ char *data = "A man who carries a cat by the tail learns " "something he can learn in no other way."; @@ -91,7 +92,7 @@ main() /* Select space in file to write a record. */ if ((file_spaceid = H5Dget_space(datasetid)) < 0) ERR; if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; /* Select space in memory to read from. */ if ((mem_spaceid = H5Screate_simple(NDIMS, count, NULL)) < 0) ERR; @@ -126,7 +127,7 @@ main() hid_t typeid, datasetid, plistid; hid_t file_spaceid, mem_spaceid; hsize_t dims[1] = {2}, chunk_dims[1] = {1}; - hsize_t start[NDIMS] = {1}, count[NDIMS] = {1}; + hsize_t start[NDIMS] = {1}, count[NDIMS] = {1}, ones[NDIMS] = {1}; /* void *fillp;*/ char *data = "A man who carries a cat by the tail learns " "something he can learn in no other way."; @@ -179,7 +180,7 @@ To be good is noble; but to show others how to be good is nobler and no trouble. /* Select space in file to write a record. */ if ((file_spaceid = H5Dget_space(datasetid)) < 0) ERR; if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; /* Select space in memory to read from. */ if ((mem_spaceid = H5Screate_simple(NDIMS, count, NULL)) < 0) diff --git a/h5_test/tst_h_vars.c b/h5_test/tst_h_vars.c index b6e2294d8f..58ea5c36e3 100644 --- a/h5_test/tst_h_vars.c +++ b/h5_test/tst_h_vars.c @@ -69,7 +69,7 @@ main() float float_data_out[LAT_LEN][LON_LEN]; hsize_t dims[NDIMS], max_dims[NDIMS]; hsize_t dims_in[NDIMS], max_dims_in[NDIMS]; - hsize_t start[MAX_DIMS], count[MAX_DIMS]; + hsize_t start[MAX_DIMS], count[MAX_DIMS], ones[MAX_DIMS]; int lat, lon; /* Set up some phoney data, 1 record's worth. In C, first @@ -153,8 +153,9 @@ main() start[0] = 1; start[1] = 0; start[2] = 0; + ones[0] = ones[1] = ones[2] = 1; if (H5Sselect_hyperslab(write_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; /* Write second record of data to each dataset. */ if (H5Dwrite(pres_dsid, H5T_IEEE_F32LE, mem_spaceid, write_spaceid, diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 0dfd9ef808..5daaac532d 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1541,11 +1541,12 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, NC_VAR_INFO_T *var; NC_DIM_INFO_T *dim; NC_HDF5_VAR_INFO_T *hdf5_var; + herr_t herr; hid_t file_spaceid = 0, mem_spaceid = 0, xfer_plistid = 0; long long unsigned xtend_size[NC_MAX_VAR_DIMS]; hsize_t fdims[NC_MAX_VAR_DIMS], fmaxdims[NC_MAX_VAR_DIMS]; hsize_t start[NC_MAX_VAR_DIMS], count[NC_MAX_VAR_DIMS]; - hsize_t stride[NC_MAX_VAR_DIMS]; + hsize_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS]; int need_to_extend = 0; #ifdef USE_PARALLEL4 int extend_possible = 0; @@ -1596,6 +1597,7 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp ? countp[i] : var->dim[i]->len; stride[i] = stridep ? stridep[i] : 1; + ones[i] = 1; LOG((4, "start[%d] %ld count[%d] %ld stride[%d] %ld", i, start[i], i, count[i], i, stride[i])); /* Check to see if any counts are zero. */ @@ -1646,8 +1648,13 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, } else { - if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, stride, - count, NULL) < 0) + if (stridep == NULL) + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, + NULL, ones, count); + else + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, + stride, count, NULL); + if (herr < 0) BAIL(NC_EHDFERR); /* Create a space for the memory, just big enough to hold the slab @@ -1772,8 +1779,14 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, BAIL2(NC_EHDFERR); if ((file_spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0) BAIL(NC_EHDFERR); - if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, stride, count, NULL) < 0) + + if (stridep == NULL) + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, NULL, ones, count); + else + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, stride, count, NULL); + if (herr < 0) BAIL(NC_EHDFERR); } } @@ -1872,7 +1885,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, hsize_t count[NC_MAX_VAR_DIMS]; hsize_t fdims[NC_MAX_VAR_DIMS], fmaxdims[NC_MAX_VAR_DIMS]; hsize_t start[NC_MAX_VAR_DIMS]; - hsize_t stride[NC_MAX_VAR_DIMS]; + hsize_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS]; void *fillvalue = NULL; int no_read = 0, provide_fill = 0; hssize_t fill_value_size[NC_MAX_VAR_DIMS]; @@ -1920,6 +1933,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp[i]; stride[i] = stridep ? stridep[i] : 1; + ones[i] = 1; /* if any of the count values are zero don't actually read. */ if (count[i] == 0) @@ -2049,9 +2063,16 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, } else { - if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, stride, count, NULL) < 0) + herr_t herr; + if (stridep == NULL) + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, NULL, ones, count); + else + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, stride, count, NULL); + if (herr < 0) BAIL(NC_EHDFERR); + /* Create a space for the memory, just big enough to hold the slab we want. */ if ((mem_spaceid = H5Screate_simple(var->ndims, count, NULL)) < 0) diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index da4ebba6f4..90ff0c0c90 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -1542,7 +1542,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, #endif size64_t fdims[NC_MAX_VAR_DIMS]; size64_t start[NC_MAX_VAR_DIMS], count[NC_MAX_VAR_DIMS]; - size64_t stride[NC_MAX_VAR_DIMS]; + size64_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS]; int retval, range_error = 0, i, d2; void *bufr = NULL; int bufrd = 0; /* 1 => we allocated bufr */ @@ -1584,6 +1584,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[0] = 0; count[0] = 1; stride[0] = 1; + ones[0] = 1; } else { for (i = 0; i < var->ndims; i++) { @@ -1594,6 +1595,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp ? countp[i] : var->dim[i]->len; stride[i] = stridep ? stridep[i] : 1; + ones[i] = 1; /* Check to see if any counts are zero. */ if (!count[i]) @@ -1649,7 +1651,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, else { if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, stride, - count, NULL) < 0) + ones, count) < 0) BAIL(NC_EHDFERR); /* Create a space for the memory, just big enough to hold the slab @@ -1757,7 +1759,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, if ((file_spaceid = H5Dget_space(ncz_var->hdf_datasetid)) < 0) BAIL(NC_EHDFERR); if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, stride, count, NULL) < 0) + start, stride, ones, count) < 0) BAIL(NC_EHDFERR); } #endif @@ -1864,6 +1866,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, size64_t fmaxdims[NC_MAX_VAR_DIMS]; size64_t start[NC_MAX_VAR_DIMS]; size64_t stride[NC_MAX_VAR_DIMS]; + size64_t ones[NC_MAX_VAR_DIMS]; int no_read = 0, provide_fill = 0; int fill_value_size[NC_MAX_VAR_DIMS]; int retval, range_error = 0, i, d2; @@ -1898,6 +1901,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[0] = 0; count[0] = 1; stride[0] = 1; + ones[0] = 1; } else { for (i = 0; i < var->ndims; i++) { @@ -1907,6 +1911,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp[i]; stride[i] = stridep ? stridep[i] : 1; + ones[i] = 1; /* if any of the count values are zero don't actually read. */ if (count[i] == 0) no_read++; @@ -2040,7 +2045,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, else { if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, stride, count, NULL) < 0) + start, stride, ones, count) < 0) BAIL(NC_EHDFERR); /* Create a space for the memory, just big enough to hold the slab we want. */ diff --git a/nc_perf/tst_files3.c b/nc_perf/tst_files3.c index ab04f7ec06..f4404e0535 100644 --- a/nc_perf/tst_files3.c +++ b/nc_perf/tst_files3.c @@ -100,6 +100,7 @@ int dump_hdf_file(const float *data, int docompression) hsize_t dims[NDIMS] = {X_LEN, Y_LEN, Z_LEN}; hsize_t start[NDIMS] = {0, 0, 0}; hsize_t count[NDIMS] = {1, 1, Z_LEN}; + hsize_t ones[NDIMS] = {1, 1, 1}; /* create file */ file_id = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, @@ -138,7 +139,7 @@ int dump_hdf_file(const float *data, int docompression) for (start[1] = 0; start[1] < Y_LEN; start[1]++) { if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL, - count, NULL) < 0) ERR_RET; + ones, count) < 0) ERR_RET; if (H5Dwrite(dataset_id, H5T_NATIVE_FLOAT, mem_spaceid, file_spaceid, xfer_plistid, data) < 0) ERR_RET; } From 66dba3f3e85dfe02db7ba07d7adb1fd4c5001d2d Mon Sep 17 00:00:00 2001 From: Greg Sjaardema Date: Wed, 6 Jul 2022 10:37:02 -0600 Subject: [PATCH 03/77] Minor fix to doxygen documentation --- libdispatch/dparallel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libdispatch/dparallel.c b/libdispatch/dparallel.c index 6a9aba720c..f13031e562 100644 --- a/libdispatch/dparallel.c +++ b/libdispatch/dparallel.c @@ -331,7 +331,7 @@ nc_open_par_fortran(const char *path, int omode, int comm, @return ::NC_NOERR No error. @return ::NC_EBADID Invalid ncid passed. @return ::NC_ENOTVAR Invalid varid passed. - @return ::NC_ENOPAR File was not opened with nc_open_par/nc_create_var. + @return ::NC_ENOPAR File was not opened with nc_open_par/nc_create_par. @return ::NC_EINVAL Invalid par_access specified, or attempt to set filtered variable to independent access. From fe3cc6fc449589ca76cf5e723aa6b31d3d4e1c3b Mon Sep 17 00:00:00 2001 From: John Parent Date: Tue, 24 Jan 2023 15:07:39 -0500 Subject: [PATCH 04/77] Find MPI during CMake package import When MPI::MPI_C is included in link interface of exported netcdf::netcdf target the config module is responsible for locating that target and providing it to the consuming project via find_dependency --- CMakeLists.txt | 2 ++ netCDFConfig.cmake.in | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b0573111b..f3a5262322 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1471,6 +1471,7 @@ ENDIF() # Enable Parallel IO with netCDF-4/HDF5 files using HDF5 parallel I/O. SET(STATUS_PARALLEL "OFF") +set(IMPORT_MPI "") OPTION(ENABLE_PARALLEL4 "Build netCDF-4 with parallel IO" "${HDF5_PARALLEL}") IF(ENABLE_PARALLEL4 AND ENABLE_HDF5) IF(NOT HDF5_PARALLEL) @@ -1492,6 +1493,7 @@ IF(ENABLE_PARALLEL4 AND ENABLE_HDF5) FILE(COPY "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh" DESTINATION ${netCDF_BINARY_DIR}/h5_test FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + set(IMPORT_MPI "include(CMakeFindDependencyMacro)\nfind_dependency(MPI COMPONENTS C)") ENDIF() ENDIF() diff --git a/netCDFConfig.cmake.in b/netCDFConfig.cmake.in index 9d68eec5a2..dae2429e4a 100644 --- a/netCDFConfig.cmake.in +++ b/netCDFConfig.cmake.in @@ -14,6 +14,8 @@ set(netCDF_LIBRARIES netCDF::netcdf) # include target information include("${CMAKE_CURRENT_LIST_DIR}/netCDFTargets.cmake") +@IMPORT_MPI@ + # Compiling Options # set(netCDF_C_COMPILER "@CC_VERSION@") From ce3bb54d555afcaef48c51687c9f65e37d899bea Mon Sep 17 00:00:00 2001 From: John Parent Date: Tue, 24 Jan 2023 15:16:55 -0500 Subject: [PATCH 05/77] If using MPI, should link against MPI target Failing to do this results in unresolved symbols during link time. --- liblib/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/liblib/CMakeLists.txt b/liblib/CMakeLists.txt index e3eddc0fb1..3df7918675 100644 --- a/liblib/CMakeLists.txt +++ b/liblib/CMakeLists.txt @@ -52,6 +52,10 @@ IF(MPI_C_INCLUDE_PATH) target_include_directories(netcdf PUBLIC ${MPI_C_INCLUDE_PATH}) ENDIF(MPI_C_INCLUDE_PATH) +IF(TARGET MPI::MPI_C) + target_link_libraries(netcdf PUBLIC MPI::MPI_C) +ENDIF(TARGET MPI::MPI_C) + IF(MOD_NETCDF_NAME) SET_TARGET_PROPERTIES(netcdf PROPERTIES LIBRARY_OUTPUT_NAME ${NETCDF_LIB_NAME}) SET_TARGET_PROPERTIES(netcdf PROPERTIES ARCHIVE_OUTPUT_NAME ${NETCDF_LIB_NAME}) From 96c7d58f04f7cea9d9c8bd95127f66a99a73b3c4 Mon Sep 17 00:00:00 2001 From: John Parent Date: Tue, 24 Jan 2023 15:18:11 -0500 Subject: [PATCH 06/77] Plug dependencies should include MPI Plugin deps should include MPI if MPI is being used in the build. This is at least the case of HDF5 --- plugins/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 65891d82ee..15567c8f6b 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -62,6 +62,9 @@ MACRO(buildplugin TARGET TARGETLIB) set_target_properties(${TARGET} PROPERTIES LINK_FLAGS "/INCREMENTAL:NO /DEBUG /OPT:REF /OPT:ICF") # Set file name & location set_target_properties(${TARGET} PROPERTIES COMPILE_PDB_NAME ${TARGET} COMPILE_PDB_OUTPUT_DIR ${CMAKE_BINARY_DIR}) + IF(MPI_C_INCLUDE_PATH) + target_include_directories(${TARGET} PRIVATE ${MPI_C_INCLUDE_PATH}) + ENDIF(MPI_C_INCLUDE_PATH) ENDIF() ENDMACRO() From f8a34e930b714d2c535fba9b8f8395c84b54fb1d Mon Sep 17 00:00:00 2001 From: John Parent Date: Wed, 25 Jan 2023 16:41:27 -0500 Subject: [PATCH 07/77] Linux: consistent use of blank link vis --- liblib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liblib/CMakeLists.txt b/liblib/CMakeLists.txt index 3df7918675..8fdfd75614 100644 --- a/liblib/CMakeLists.txt +++ b/liblib/CMakeLists.txt @@ -53,7 +53,7 @@ IF(MPI_C_INCLUDE_PATH) ENDIF(MPI_C_INCLUDE_PATH) IF(TARGET MPI::MPI_C) - target_link_libraries(netcdf PUBLIC MPI::MPI_C) + target_link_libraries(netcdf MPI::MPI_C) ENDIF(TARGET MPI::MPI_C) IF(MOD_NETCDF_NAME) From 31cc60e804403b42eaea71ccc1780d09ed02f469 Mon Sep 17 00:00:00 2001 From: DWesl <22566757+DWesl@users.noreply.github.com> Date: Mon, 30 Jan 2023 14:04:03 -0500 Subject: [PATCH 08/77] CI: Have nc-autotools use source distribution (#1) Instead of a clone of the repository, have the nc-autotools job work from a source distribution prepared by a previous autotools CI job. This should catch most of the "files not included in EXTRA_DIST" or similar issues I remember, and probably most of the "netcdf-c does not pass make distcheck" errors. --- .github/workflows/run_tests_ubuntu.yml | 92 ++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 7 deletions(-) diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 816367990b..9909e0aa15 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -195,6 +195,19 @@ jobs: run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j if: ${{ success() }} + - name: Create source distribution + shell: bash -l {0} + if: ${{ success() }} + run: make dist -j + + - uses: actions/upload-artifact@v3 + with: + name: netcdf-c-autotools-source-distribution + path: | + *.tar* + *.zip + *.tgz + ## # Parallel ## @@ -448,7 +461,28 @@ jobs: use_nczarr: [ nczarr_off, nczarr_on ] steps: - - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: + name: netcdf-c-autotools-source-distribution + + - name: Unpack source distribution + shell: bash -l {0} + run: | + if [ -f *.zip ]; + then + unzip *.zip + else + tar xvzf $(ls *.tar* *.tgz *.zip | head -1) + fi + ls -d netcdf-c* + for name in netcdf-c*; + do + if [ -d ${name} ]; + then + cd ${name} + break + fi + done - name: Install System dependencies shell: bash -l {0} @@ -497,11 +531,28 @@ jobs: - name: Run autoconf shell: bash -l {0} - run: autoreconf -if + run: | + for name in netcdf-c*; + do + if [ -d ${name} ]; + then + cd ${name} + break + fi + done + autoreconf -if - name: Configure shell: bash -l {0} run: | + for name in netcdf-c*; + do + if [ -d ${name} ]; + then + cd ${name} + break + fi + done current_directory="$(pwd)" mkdir ../build cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} "${current_directory}/configure" ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR} @@ -509,29 +560,56 @@ jobs: - name: Look at config.log if error shell: bash -l {0} - run: cd ../build && cat config.log + run: | + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && cat config.log if: ${{ failure() }} - name: Print Summary shell: bash -l {0} - run: cd ../build && cat libnetcdf.settings + run: | + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && cat libnetcdf.settings - name: Build Library and Utilities shell: bash -l {0} run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j if: ${{ success() }} - name: Build Tests shell: bash -l {0} run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j if: ${{ success() }} - name: Run Tests shell: bash -l {0} run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j + if [ -d ../build ]; + then + cd ../build + else + cd build + fi && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j if: ${{ success() }} nc-cmake: From d755c4ff32cabe2b1862c6ad66f45bb32341ef28 Mon Sep 17 00:00:00 2001 From: Eisuke Kawashima Date: Wed, 23 Aug 2023 13:28:53 +0900 Subject: [PATCH 09/77] chore: unset executable flag --- cmake/modules/FindBlosc.cmake | 0 cmake/modules/FindZip.cmake | 0 docs/testserver.dox | 0 libdispatch/dcrc32.h | 0 libncpoco/CMakeLists.txt | 0 libncpoco/COPYRIGHT | 0 libncpoco/Makefile.am | 0 libncpoco/cp_test.c | 0 libncpoco/cp_unix.c | 0 libncpoco/cp_win32.c | 0 libncpoco/cptestlib.c | 0 libncpoco/ncpoco.c | 0 libncpoco/ncpoco.h | 0 libnczarr/zmap_file.c | 0 libnczarr/zmap_zip.c | 0 nc_test4/h5testszip.c | 0 nc_test4/ref_fixedstring.h5 | Bin nc_test4/tst_charvlenbug.c | 0 nc_test4/tst_fillonly.c | 0 ncdap_test/manyurls.h | 0 ncdump/ncpathcvt.c | 0 ncdump/ocprint.c | 0 ncdump/tst_rcapi.c | 0 nczarr_test/ncdumpchunks.c | 0 nczarr_test/ref_groups.h5 | Bin nczarr_test/ut_chunking.c | 0 nczarr_test/ut_projtest.h | 0 nczarr_test/ut_test.c | 0 plugins/H5Zblosc.c | 0 plugins/H5Zblosc.h | 0 plugins/H5Zszip.h | 0 31 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 cmake/modules/FindBlosc.cmake mode change 100755 => 100644 cmake/modules/FindZip.cmake mode change 100755 => 100644 docs/testserver.dox mode change 100755 => 100644 libdispatch/dcrc32.h mode change 100755 => 100644 libncpoco/CMakeLists.txt mode change 100755 => 100644 libncpoco/COPYRIGHT mode change 100755 => 100644 libncpoco/Makefile.am mode change 100755 => 100644 libncpoco/cp_test.c mode change 100755 => 100644 libncpoco/cp_unix.c mode change 100755 => 100644 libncpoco/cp_win32.c mode change 100755 => 100644 libncpoco/cptestlib.c mode change 100755 => 100644 libncpoco/ncpoco.c mode change 100755 => 100644 libncpoco/ncpoco.h mode change 100755 => 100644 libnczarr/zmap_file.c mode change 100755 => 100644 libnczarr/zmap_zip.c mode change 100755 => 100644 nc_test4/h5testszip.c mode change 100755 => 100644 nc_test4/ref_fixedstring.h5 mode change 100755 => 100644 nc_test4/tst_charvlenbug.c mode change 100755 => 100644 nc_test4/tst_fillonly.c mode change 100755 => 100644 ncdap_test/manyurls.h mode change 100755 => 100644 ncdump/ncpathcvt.c mode change 100755 => 100644 ncdump/ocprint.c mode change 100755 => 100644 ncdump/tst_rcapi.c mode change 100755 => 100644 nczarr_test/ncdumpchunks.c mode change 100755 => 100644 nczarr_test/ref_groups.h5 mode change 100755 => 100644 nczarr_test/ut_chunking.c mode change 100755 => 100644 nczarr_test/ut_projtest.h mode change 100755 => 100644 nczarr_test/ut_test.c mode change 100755 => 100644 plugins/H5Zblosc.c mode change 100755 => 100644 plugins/H5Zblosc.h mode change 100755 => 100644 plugins/H5Zszip.h diff --git a/cmake/modules/FindBlosc.cmake b/cmake/modules/FindBlosc.cmake old mode 100755 new mode 100644 diff --git a/cmake/modules/FindZip.cmake b/cmake/modules/FindZip.cmake old mode 100755 new mode 100644 diff --git a/docs/testserver.dox b/docs/testserver.dox old mode 100755 new mode 100644 diff --git a/libdispatch/dcrc32.h b/libdispatch/dcrc32.h old mode 100755 new mode 100644 diff --git a/libncpoco/CMakeLists.txt b/libncpoco/CMakeLists.txt old mode 100755 new mode 100644 diff --git a/libncpoco/COPYRIGHT b/libncpoco/COPYRIGHT old mode 100755 new mode 100644 diff --git a/libncpoco/Makefile.am b/libncpoco/Makefile.am old mode 100755 new mode 100644 diff --git a/libncpoco/cp_test.c b/libncpoco/cp_test.c old mode 100755 new mode 100644 diff --git a/libncpoco/cp_unix.c b/libncpoco/cp_unix.c old mode 100755 new mode 100644 diff --git a/libncpoco/cp_win32.c b/libncpoco/cp_win32.c old mode 100755 new mode 100644 diff --git a/libncpoco/cptestlib.c b/libncpoco/cptestlib.c old mode 100755 new mode 100644 diff --git a/libncpoco/ncpoco.c b/libncpoco/ncpoco.c old mode 100755 new mode 100644 diff --git a/libncpoco/ncpoco.h b/libncpoco/ncpoco.h old mode 100755 new mode 100644 diff --git a/libnczarr/zmap_file.c b/libnczarr/zmap_file.c old mode 100755 new mode 100644 diff --git a/libnczarr/zmap_zip.c b/libnczarr/zmap_zip.c old mode 100755 new mode 100644 diff --git a/nc_test4/h5testszip.c b/nc_test4/h5testszip.c old mode 100755 new mode 100644 diff --git a/nc_test4/ref_fixedstring.h5 b/nc_test4/ref_fixedstring.h5 old mode 100755 new mode 100644 diff --git a/nc_test4/tst_charvlenbug.c b/nc_test4/tst_charvlenbug.c old mode 100755 new mode 100644 diff --git a/nc_test4/tst_fillonly.c b/nc_test4/tst_fillonly.c old mode 100755 new mode 100644 diff --git a/ncdap_test/manyurls.h b/ncdap_test/manyurls.h old mode 100755 new mode 100644 diff --git a/ncdump/ncpathcvt.c b/ncdump/ncpathcvt.c old mode 100755 new mode 100644 diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c old mode 100755 new mode 100644 diff --git a/ncdump/tst_rcapi.c b/ncdump/tst_rcapi.c old mode 100755 new mode 100644 diff --git a/nczarr_test/ncdumpchunks.c b/nczarr_test/ncdumpchunks.c old mode 100755 new mode 100644 diff --git a/nczarr_test/ref_groups.h5 b/nczarr_test/ref_groups.h5 old mode 100755 new mode 100644 diff --git a/nczarr_test/ut_chunking.c b/nczarr_test/ut_chunking.c old mode 100755 new mode 100644 diff --git a/nczarr_test/ut_projtest.h b/nczarr_test/ut_projtest.h old mode 100755 new mode 100644 diff --git a/nczarr_test/ut_test.c b/nczarr_test/ut_test.c old mode 100755 new mode 100644 diff --git a/plugins/H5Zblosc.c b/plugins/H5Zblosc.c old mode 100755 new mode 100644 diff --git a/plugins/H5Zblosc.h b/plugins/H5Zblosc.h old mode 100755 new mode 100644 diff --git a/plugins/H5Zszip.h b/plugins/H5Zszip.h old mode 100755 new mode 100644 From e6d9cda0e91b2d8bd0d5f29c4af7685c07219200 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 6 Sep 2023 14:43:53 -0400 Subject: [PATCH 10/77] netCDFConfig: find HDF5 if needed See: #1733 --- netCDFConfig.cmake.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/netCDFConfig.cmake.in b/netCDFConfig.cmake.in index 9d68eec5a2..0cd843db1e 100644 --- a/netCDFConfig.cmake.in +++ b/netCDFConfig.cmake.in @@ -40,3 +40,8 @@ set(netCDF_HAS_DAP4 @HAS_DAP4@) set(netCDF_HAS_DISKLESS @HAS_DISKLESS@) set(netCDF_HAS_MMAP @HAS_MMAP@) set(netCDF_HAS_JNA @HAS_JNA@) + +if (netCDF_HAS_HDF4 OR netCDF_HAS_HDF5) + include(CMakeFindDependencyMacro) + find_dependency(HDF5) +endif () From 375e5105ff8c7bc331a615fb25ca8ca8d78b50bd Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Mon, 2 Oct 2023 10:14:18 -0600 Subject: [PATCH 11/77] Update Release Notes --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c92394242d..50c0ea53b5 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD -* Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). +* Mitigate the problem of remote/nczarr-related test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). * Fix default parameters for caching of NCZarr. See [Github #2734](https://github.com/Unidata/netcdf-c/pull/2734). From ed53598aac70543f32c8e3b6ab458ac942d88b40 Mon Sep 17 00:00:00 2001 From: Magnus Ulimoen Date: Fri, 6 Oct 2023 13:48:00 +0200 Subject: [PATCH 12/77] Do not run test unless required --- libdispatch/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libdispatch/CMakeLists.txt b/libdispatch/CMakeLists.txt index 98b90bc96c..1d867369b8 100644 --- a/libdispatch/CMakeLists.txt +++ b/libdispatch/CMakeLists.txt @@ -51,7 +51,9 @@ IF(ENABLE_S3) ENDIF() ENDIF() -BUILD_BIN_TEST(ncrandom) +IF(ENABLE_TESTS) + BUILD_BIN_TEST(ncrandom) +ENDIF() FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.c) SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am) From 1552d894a2c8a7f58e06f24eb2a06cd614be0e66 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 8 Oct 2023 11:22:52 -0600 Subject: [PATCH 13/77] Cleanup a number of issues. re: Issue https://github.com/Unidata/netcdf-c/issues/2748 This PR fixes a number of issues and bugs. ## s3cleanup fixes * Delete extraneous s3cleanup.sh related files. * Remove duplicate s3cleanup.uids entries. ## Support the Google S3 API * Add code to recognize "storage.gooleapis.com" * Add extra code to track the kind of server being accessed: unknown, Amazon, Google. * Add a new mode flag "gs3" (analog to "s3") to support this api. * Modify the S3 URL code to support this case. * Modify the listobjects result parsing because Google returns some non-standard XML elements. * Change signature and calls for NC_s3urlrebuild. ## Handle corrupt Zarr files where shape is empty for a variable. Modify behavior when a variable's "shape" dictionary entry. Previously it returned an error, but now it suppresses such a variable. This change makes it possible to read non-corrupt data from the file. Also added a test case. ## Misc. Other Changes * Fix the nclog level handling to suppress output by default. * Fix de-duplicates code in ncuri.c * Restore testing of iridl.ldeo.columbia.edu. * Fix bug in define_vars() which did not always do a proper reclaim between variables. --- Makefile.am | 4 + include/ncrc.h | 5 +- include/ncs3sdk.h | 6 ++ libdispatch/dinfermodel.c | 4 +- libdispatch/drc.c | 10 ++- libdispatch/ds3util.c | 95 ++++++++++++++------- libdispatch/nch5s3comms.c | 16 ++-- libdispatch/nch5s3comms.h | 2 +- libdispatch/nclog.c | 2 +- libdispatch/ncs3sdk_h5.c | 16 +++- libdispatch/ncuri.c | 2 +- libhdf5/hdf5open.c | 22 ++--- libnczarr/zclose.c | 2 +- libnczarr/zdebug.c | 2 +- libnczarr/zsync.c | 138 +++++++++++++++++++------------ nc_test4/tst_bloscfail.sh | 1 - ncdap_test/tst_encode.sh | 1 - nczarr_test/CMakeLists.txt | 2 + nczarr_test/Makefile.am | 9 +- nczarr_test/ref_noshape.file.zip | Bin 0 -> 1481 bytes nczarr_test/run_corrupt.sh | 35 ++++++++ nczarr_test/run_scalar.sh | 8 +- plugins/H5Zmisc.c | 18 ++-- test_common.in | 13 +-- 24 files changed, 280 insertions(+), 133 deletions(-) create mode 100644 nczarr_test/ref_noshape.file.zip create mode 100755 nczarr_test/run_corrupt.sh diff --git a/Makefile.am b/Makefile.am index 64648d6180..55907f7ef5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -211,7 +211,11 @@ install-data-hook: all-local: liblib/libnetcdf.la echo ${PACKAGE_VERSION} > VERSION if ENABLE_S3_TESTALL + rm -f ${abs_top_builddir}/tmp_@PLATFORMUID@.uids echo "@TESTUID@" >> ${abs_top_builddir}/s3cleanup_@PLATFORMUID@.uids + cat ${abs_top_builddir}/s3cleanup_@PLATFORMUID@.uids | sort | uniq > ${abs_top_builddir}/tmp_@PLATFORMUID@.uids + rm -f ${abs_top_builddir}/s3cleanup_@PLATFORMUID@.uids + mv ${abs_top_builddir}/tmp_@PLATFORMUID@.uids ${abs_top_builddir}/s3cleanup_@PLATFORMUID@.uids endif if ENABLE_S3_TESTALL diff --git a/include/ncrc.h b/include/ncrc.h index b3bb8c512d..ae858532e7 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -53,6 +53,9 @@ typedef struct NCRCinfo { NClist* s3profiles; /* NClist */ } NCRCinfo; +/* Opaque structures */ +struct NCS3INFO; + #if defined(__cplusplus) extern "C" { #endif @@ -94,7 +97,7 @@ EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep); EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep); EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep); EXTERNL int NC_iss3(NCURI* uri); -EXTERNL int NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp); +EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp); #if defined(__cplusplus) } diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index c06f0e39f1..771faa6666 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -6,6 +6,11 @@ #ifndef NCS3SDK_H #define NCS3SDK_H 1 +/* Track the server type, if known */ +typedef enum NCS3SVC {NCS3UNK=0, /* unknown */ + NCS3=1, /* s3.amazon.aws */ + NCS3GS=0 /* storage.googleapis.com */ +} NCS3SVC; typedef struct NCS3INFO { char* host; /* non-null if other*/ @@ -13,6 +18,7 @@ typedef struct NCS3INFO { char* bucket; /* bucket name */ char* rootkey; char* profile; + NCS3SVC svc; } NCS3INFO; #ifdef __cplusplus diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 937fd8bd08..61941d1d71 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -136,6 +136,7 @@ static const struct MACRODEF { {"xarray","mode",{"zarr", NULL}}, {"noxarray","mode",{"nczarr", "noxarray", NULL}}, {"zarr","mode",{"nczarr","zarr", NULL}}, +{"gs3","mode",{"gs3","nczarr",NULL}}, /* Google S3 API */ {NULL,NULL,{NULL}} }; @@ -196,6 +197,7 @@ static struct NCPROTOCOLLIST { {"dods","http","mode=dap2"}, {"dap4","http","mode=dap4"}, {"s3","s3","mode=s3"}, + {"gs3","gs3","mode=gs3"}, {NULL,NULL,NULL} /* Terminate search */ }; @@ -914,7 +916,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void /* If s3, then rebuild the url */ if(NC_iss3(uri)) { NCURI* newuri = NULL; - if((stat = NC_s3urlrebuild(uri,NULL,NULL,&newuri))) goto done; + if((stat = NC_s3urlrebuild(uri,NULL,&newuri))) goto done; ncurifree(uri); uri = newuri; } else if(strcmp(uri->protocol,"file")==0) { diff --git a/libdispatch/drc.c b/libdispatch/drc.c index b896d90bfb..77922b752f 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -437,7 +437,9 @@ rccompile(const char* filepath) NCURI* uri = NULL; char* nextline = NULL; NCglobalstate* globalstate = NC_getglobalstate(); - char* bucket = NULL; + NCS3INFO s3; + + memset(&s3,0,sizeof(s3)); if((ret=NC_readfile(filepath,tmp))) { nclog(NCLOGWARN, "Could not open configuration file: %s",filepath); @@ -484,9 +486,8 @@ rccompile(const char* filepath) if(NC_iss3(uri)) { NCURI* newuri = NULL; /* Rebuild the url to S3 "path" format */ - nullfree(bucket); - bucket = NULL; - if((ret = NC_s3urlrebuild(uri,&bucket,NULL,&newuri))) goto done; + NC_s3clear(&s3); + if((ret = NC_s3urlrebuild(uri,&s3,&newuri))) goto done; ncurifree(uri); uri = newuri; newuri = NULL; @@ -546,6 +547,7 @@ rccompile(const char* filepath) rcorder(rc); done: + NC_s3clear(&s3); if(contents) free(contents); ncurifree(uri); ncbytesfree(tmp); diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index df001f8023..5091b1a24e 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -30,6 +30,7 @@ #undef AWSDEBUG #define AWSHOST ".amazonaws.com" +#define GOOGLEHOST "storage.googleapis.com" enum URLFORMAT {UF_NONE=0, UF_VIRTUAL=1, UF_PATH=2, UF_S3=3, UF_OTHER=4}; @@ -44,15 +45,12 @@ Rebuild an S3 url into a canonical path-style url. If region is not in the host, then use specified region if provided, otherwise us-east-1. @param url (in) the current url -@param region (in) region to use if needed; NULL => us-east-1 - (out) region from url or the input region -@param bucketp (in) bucket to use if needed - (out) bucket from url +@param s3 (in/out) NCS3INFO structure @param pathurlp (out) the resulting pathified url string */ int -NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp) +NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) { int i,stat = NC_NOERR; NClist* hostsegments = NULL; @@ -63,6 +61,7 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne char* host = NULL; char* path = NULL; char* region = NULL; + NCS3SVC svc = NCS3UNK; if(url == NULL) {stat = NC_EURL; goto done;} @@ -83,14 +82,27 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne Path: https://s3..amazonaws.com// (3) or: https://s3.amazonaws.com// -- region defaults to us-east-1 (4) S3: s3:/// (5) - Other: https://// (6) + Google: https://storage.googleapis.com// (6) + or: gs3:/// (7) + Other: https://// (8) */ if(url->host == NULL || strlen(url->host) == 0) {stat = NC_EURL; goto done;} + + /* Reduce the host to standard form such as s3.amazonaws.com by pulling out the + region and bucket from the host */ if(strcmp(url->protocol,"s3")==0 && nclistlength(hostsegments)==1) { /* Format (5) */ bucket = nclistremove(hostsegments,0); /* region unknown at this point */ + /* Host will be set to canonical form later */ + svc = NCS3; + } else if(strcmp(url->protocol,"gs3")==0 && nclistlength(hostsegments)==1) { /* Format (7) */ + bucket = nclistremove(hostsegments,0); + /* region unknown at this point */ + /* Host will be set to canonical form later */ + svc = NCS3GS; } else if(endswith(url->host,AWSHOST)) { /* Virtual or path */ + svc = NCS3; /* If we find a bucket as part of the host, then remove it */ switch (nclistlength(hostsegments)) { default: stat = NC_EURL; goto done; @@ -99,11 +111,11 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne /* bucket unknown at this point */ break; case 4: /* Format (2) or (3) */ - if(strcasecmp(nclistget(hostsegments,1),"s3")==0) { /* Format (2) */ + if(strcasecmp(nclistget(hostsegments,0),"s3")!=0) { /* Presume format (2) */ /* region unknown at this point */ - bucket = nclistremove(hostsegments,0); /* Note removeal */ + bucket = nclistremove(hostsegments,0); /* Make canonical */ } else if(strcasecmp(nclistget(hostsegments,0),"s3")==0) { /* Format (3) */ - region = strdup(nclistget(hostsegments,1)); + region = nclistremove(hostsegments,1); /* Make canonical */ /* bucket unknown at this point */ } else /* ! Format (2) and ! Format (3) => error */ {stat = NC_EURL; goto done;} @@ -111,20 +123,27 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne case 5: /* Format (1) */ if(strcasecmp(nclistget(hostsegments,1),"s3")!=0) {stat = NC_EURL; goto done;} - region = strdup(nclistget(hostsegments,2)); - bucket = strdup(nclistremove(hostsegments,0)); + /* Make canonical */ + region = nclistremove(hostsegments,2); + bucket = nclistremove(hostsegments,0); break; } - } else { /* Presume Format (6) */ + } else if(strcasecmp(url->host,GOOGLEHOST)==0) { /* Google (6) */ + if((host = strdup(url->host))==NULL) + {stat = NC_ENOMEM; goto done;} + /* region is unknown */ + /* bucket is unknown at this point */ + svc = NCS3GS; + } else { /* Presume Format (8) */ if((host = strdup(url->host))==NULL) {stat = NC_ENOMEM; goto done;} /* region is unknown */ /* bucket is unknown */ } - /* region = (1) from url, (2) inoutregion, (3) default */ - if(region == NULL) - region = (inoutregionp?nulldup(*inoutregionp):NULL); + /* region = (1) from url, (2) s3->region, (3) default */ + if(region == NULL && s3 != NULL) + region = nulldup(s3->region); if(region == NULL) { const char* region0 = NULL; /* Get default region */ @@ -133,23 +152,30 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne } if(region == NULL) {stat = NC_ES3; goto done;} - /* bucket = (1) from url, (2) inoutbucket */ + /* bucket = (1) from url, (2) s3->bucket */ if(bucket == NULL && nclistlength(pathsegments) > 0) { bucket = nclistremove(pathsegments,0); /* Get from the URL path; will reinsert below */ } - if(bucket == NULL) - bucket = (inoutbucketp?nulldup(*inoutbucketp):NULL); + if(bucket == NULL && s3 != NULL) + bucket = nulldup(s3->bucket); if(bucket == NULL) {stat = NC_ES3; goto done;} - if(host == NULL) { /* Construct the revised host */ + if(svc == NCS3) { + /* Construct the revised host */ + ncbytesclear(buf); ncbytescat(buf,"s3."); ncbytescat(buf,region); ncbytescat(buf,AWSHOST); + nullfree(host); host = ncbytesextract(buf); + } else if(svc == NCS3GS) { + nullfree(host); + host = strdup(GOOGLEHOST); } - /* Construct the revised path */ ncbytesclear(buf); + + /* Construct the revised path */ if(bucket != NULL) { ncbytescat(buf,"/"); ncbytescat(buf,bucket); @@ -159,10 +185,13 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne ncbytescat(buf,nclistget(pathsegments,i)); } path = ncbytesextract(buf); + /* complete the new url */ if((newurl=ncuriclone(url))==NULL) {stat = NC_ENOMEM; goto done;} ncurisetprotocol(newurl,"https"); + assert(host != NULL); ncurisethost(newurl,host); + assert(path != NULL); ncurisetpath(newurl,path); /* Rebuild the url->url */ ncurirebuild(newurl); @@ -171,9 +200,11 @@ NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** ne fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=%s region=%s\n",uri->uri,bucket,region); #endif if(newurlp) {*newurlp = newurl; newurl = NULL;} - if(inoutbucketp) {*inoutbucketp = bucket; bucket = NULL;} - if(inoutregionp) {*inoutregionp = region; region = NULL;} - + if(s3 != NULL) { + s3->bucket = bucket; bucket = NULL; + s3->region = region; region = NULL; + s3->svc = svc; + } done: nullfree(region); nullfree(bucket) @@ -218,7 +249,7 @@ NC_s3urlprocess(NCURI* url, NCS3INFO* s3, NCURI** newurlp) s3->profile = strdup(profile0); /* Rebuild the URL to path format and get a usable region and optional bucket*/ - if((stat = NC_s3urlrebuild(url,&s3->bucket,&s3->region,&url2))) goto done; + if((stat = NC_s3urlrebuild(url,s3,&url2))) goto done; s3->host = strdup(url2->host); /* construct the rootkey minus the leading bucket */ pathsegments = nclistnew(); @@ -268,7 +299,7 @@ NC_s3clear(NCS3INFO* s3) } /* -Check if a url has indicators that signal an S3 url. +Check if a url has indicators that signal an S3 or Google S3 url. */ int @@ -277,13 +308,17 @@ NC_iss3(NCURI* uri) int iss3 = 0; if(uri == NULL) goto done; /* not a uri */ - /* is the protocol "s3"? */ + /* is the protocol "s3" or "gs3" ? */ if(strcasecmp(uri->protocol,"s3")==0) {iss3 = 1; goto done;} - /* Is "s3" in the mode list? */ - if(NC_testmode(uri,"s3")) {iss3 = 1; goto done;} + if(strcasecmp(uri->protocol,"gs3")==0) {iss3 = 1; goto done;} + /* Is "s3" or "gs3" in the mode list? */ + if(NC_testmode(uri,"s3")) {iss3 = 1; goto done;} + if(NC_testmode(uri,"gs3")) {iss3 = 1; goto done;} /* Last chance; see if host looks s3'y */ - if(endswith(uri->host,AWSHOST)) {iss3 = 1; goto done;} - + if(uri->host != NULL) { + if(endswith(uri->host,AWSHOST)) {iss3 = 1; goto done;} + if(strcasecmp(uri->host,GOOGLEHOST)==0) {iss3 = 1; goto done;} + } done: return iss3; } diff --git a/libdispatch/nch5s3comms.c b/libdispatch/nch5s3comms.c index fd473f64e5..ba09b931b0 100644 --- a/libdispatch/nch5s3comms.c +++ b/libdispatch/nch5s3comms.c @@ -96,6 +96,7 @@ /*****************/ +#include "ncs3sdk.h" #include "nch5s3comms.h" /* S3 Communications */ /****************/ @@ -1063,7 +1064,7 @@ NCH5_s3comms_s3r_execute(s3r_t *handle, const char* url, *---------------------------------------------------------------------------- */ s3r_t * -NCH5_s3comms_s3r_open(const char* root, const char *region, const char *access_id, const char* access_key) +NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char *region, const char *access_id, const char* access_key) { int ret_value = SUCCEED; size_t tmplen = 0; @@ -1092,10 +1093,15 @@ NCH5_s3comms_s3r_open(const char* root, const char *region, const char *access_i * RECORD THE ROOT PATH *************************************/ - /* Verify that the region is a substring of root */ - if(region != NULL && region[0] != '\0') { - if(strstr(root,region) == NULL) - HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "region not present in root path."); + switch (svc) { + case NCS3: + /* Verify that the region is a substring of root */ + if(region != NULL && region[0] != '\0') { + if(strstr(root,region) == NULL) + HGOTO_ERROR(H5E_ARGS, NC_EINVAL, NULL, "region not present in root path."); + } + break; + default: break; } handle->rootpath = nulldup(root); diff --git a/libdispatch/nch5s3comms.h b/libdispatch/nch5s3comms.h index acfffb2858..7cc482df66 100644 --- a/libdispatch/nch5s3comms.h +++ b/libdispatch/nch5s3comms.h @@ -502,7 +502,7 @@ EXTERNL hrb_t *NCH5_s3comms_hrb_init_request(const char *resource, const char *h * DECLARATION OF S3REQUEST ROUTINES * *************************************/ -EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, const char* region, const char* id, const char* access_key); +EXTERNL s3r_t *NCH5_s3comms_s3r_open(const char* root, NCS3SVC svc, const char* region, const char* id, const char* access_key); EXTERNL int NCH5_s3comms_s3r_close(s3r_t *handle); diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index 26472fdfe0..9b9d35f0f5 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -136,7 +136,7 @@ ncvlog(int level, const char* fmt, va_list ap) const char* prefix; if(!nclogginginitialized) ncloginit(); - if(nclog_global.loglevel > level || nclog_global.nclogstream == NULL) + if(nclog_global.loglevel < level) return; prefix = nctagname(level); fprintf(nclog_global.nclogstream,"%s: ",prefix); diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c index e9da587f11..e51f8d8110 100644 --- a/libdispatch/ncs3sdk_h5.c +++ b/libdispatch/ncs3sdk_h5.c @@ -15,8 +15,8 @@ #include "ncrc.h" #include "ncxml.h" -#include "nch5s3comms.h" #include "ncs3sdk.h" +#include "nch5s3comms.h" #define NCTRACING #ifdef NCTRACING @@ -181,7 +181,7 @@ NC_s3sdkcreateclient(NCS3INFO* info) if((stat = NC_s3profilelookup(info->profile, "aws_secret_access_key", &accesskey))) goto done; } if((s3client->rooturl = makes3rooturl(info))==NULL) {stat = NC_ENOMEM; goto done;} - s3client->h5s3client = NCH5_s3comms_s3r_open(s3client->rooturl,info->region,accessid,accesskey); + s3client->h5s3client = NCH5_s3comms_s3r_open(s3client->rooturl,info->svc,info->region,accessid,accesskey); if(s3client->h5s3client == NULL) {stat = NC_ES3; goto done;} done: @@ -613,6 +613,10 @@ HTTP/1.1 200 string string +#ifdef GOOGLES3 + string + string +#endif ... ... @@ -679,6 +683,8 @@ parse_listbucketresult(char* xml, unsigned long long xmllen, struct LISTOBJECTSV result->nextcontinuationtoken = trim(ncxml_text(x),RECLAIM); } else if(strcmp(elem,"StartAfter")==0) { result->startafter = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"StartAfter")==0) { + result->startafter = trim(ncxml_text(x),RECLAIM); } else { nclog(NCLOGERR,"Unexpected Element: <%s>",elem); stat = NC_ES3; @@ -711,7 +717,7 @@ parse_object(ncxml_t root, NClist* objects) for(x=ncxml_child_first(root);x != NULL;x=ncxml_child_next(x)) { const char* elem = ncxml_name(x); - if(strcmp(elem,"ChecksumAlorithm")==0) { + if(strcmp(elem,"ChecksumAlgorithm")==0) { if((stat = parse_checksumalgorithm(x,object->checksumalgorithms))) goto done; } else if(strcmp(elem,"ETag")==0) { object->etag = trim(ncxml_text(x),RECLAIM); @@ -725,6 +731,10 @@ parse_object(ncxml_t root, NClist* objects) object->size = trim(ncxml_text(x),RECLAIM); } else if(strcmp(elem,"StorageClass")==0) { object->storageclass = trim(ncxml_text(x),RECLAIM); + } else if(strcmp(elem,"Generation")==0) { + /* Ignore */ + } else if(strcmp(elem,"MetaGeneration")==0) { + /* Ignore */ } else { nclog(NCLOGERR,"Unexpected Element: <%s>",elem); stat = NC_ES3; diff --git a/libdispatch/ncuri.c b/libdispatch/ncuri.c index 7bca11a228..efd6fa4492 100644 --- a/libdispatch/ncuri.c +++ b/libdispatch/ncuri.c @@ -1237,7 +1237,7 @@ removedups(NClist* list) /* look for dups for this entry */ for(j=nclistlength(list)-2;j>i;j-=2) { if(strcasecmp(nclistget(list,i),nclistget(list,j))==0 - && strcasecmp(nclistget(list,i+1),nclistget(list,j+1))) { + && strcasecmp(nclistget(list,i+1),nclistget(list,j+1))==0) { nclistremove(list,j+1); nclistremove(list,j); } } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 514d1e2ffb..c63e649d9e 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -25,6 +25,7 @@ #ifdef ENABLE_HDF5_ROS3 #include +#include "ncs3sdk.h" #endif /*Nemonic */ @@ -883,12 +884,11 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) #ifdef ENABLE_BYTERANGE else if(h5->byterange) { /* Arrange to use the byte-range drivers */ char* newpath = NULL; - char* awsregion0 = NULL; #ifdef ENABLE_HDF5_ROS3 H5FD_ros3_fapl_t fa; - const char* profile0 = NULL; const char* awsaccessid0 = NULL; const char* awssecretkey0 = NULL; + const char* profile0 = NULL; int iss3 = NC_iss3(h5->uri); fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION; @@ -898,9 +898,11 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) fa.secret_key[0] = '\0'; if(iss3) { - /* Rebuild the URL */ + NCS3INFO s3; NCURI* newuri = NULL; - if((retval = NC_s3urlrebuild(h5->uri,NULL,&awsregion0,&newuri))) goto exit; + /* Rebuild the URL */ + memset(&s3,0,sizeof(s3)); + if((retval = NC_s3urlrebuild(h5->uri,&s3,&newuri))) goto exit; if((newpath = ncuribuild(newuri,NULL,NULL,NCURISVC))==NULL) {retval = NC_EURL; goto exit;} ncurifree(h5->uri); @@ -909,22 +911,23 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) BAIL(retval); if((retval = NC_s3profilelookup(profile0,AWS_ACCESS_KEY_ID,&awsaccessid0))) BAIL(retval); - if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0))) + if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0))) BAIL(retval); - if(awsregion0 == NULL) - awsregion0 = strdup(S3_REGION_DEFAULT); + if(s3.region == NULL) + s3.region = strdup(S3_REGION_DEFAULT); if(awsaccessid0 == NULL || awssecretkey0 == NULL ) { /* default, non-authenticating, "anonymous" fapl configuration */ fa.authenticate = (hbool_t)0; } else { fa.authenticate = (hbool_t)1; - assert(awsregion0 != NULL && strlen(awsregion0) > 0); + assert(s3.region != NULL && strlen(s3.region) > 0); assert(awsaccessid0 != NULL && strlen(awsaccessid0) > 0); assert(awssecretkey0 != NULL && strlen(awssecretkey0) > 0); - strlcat(fa.aws_region,awsregion0,H5FD_ROS3_MAX_REGION_LEN); + strlcat(fa.aws_region,s3.region,H5FD_ROS3_MAX_REGION_LEN); strlcat(fa.secret_id, awsaccessid0, H5FD_ROS3_MAX_SECRET_ID_LEN); strlcat(fa.secret_key, awssecretkey0, H5FD_ROS3_MAX_SECRET_KEY_LEN); } + NC_s3clear(&s3); /* create and set fapl entry */ if(H5Pset_fapl_ros3(fapl_id, &fa) < 0) BAIL(NC_EHDFERR); @@ -938,7 +941,6 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) if ((h5->hdfid = nc4_H5Fopen((newpath?newpath:path), flags, fapl_id)) < 0) BAIL(NC_EHDFERR); nullfree(newpath); - nullfree(awsregion0); } #endif else { diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index b70e86f794..fdef3f5c16 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -179,7 +179,7 @@ NCZ_zclose_var1(NC_VAR_INFO_T* var) } /** - * @internal Close resources for vars in a group. + * @internal Close nczarr resources for vars in a group. * * @param grp Pointer to group info struct. * diff --git a/libnczarr/zdebug.c b/libnczarr/zdebug.c index 2a1ad4efaf..3b1e0e9217 100644 --- a/libnczarr/zdebug.c +++ b/libnczarr/zdebug.c @@ -34,7 +34,7 @@ int zreport(int err, const char* msg, const char* file, const char* fcn, int line) { if(err == 0) return err; - ZLOG(NCLOGWARN,"!!! zreport: err=%d msg=%s",err,msg); + ZLOG(NCLOGWARN,"!!! zreport: err=%d msg=%s @ %s#%s:%d",err,msg,file,fcn,line); ncbacktrace(); return zbreakpoint(err); } diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index 0d2596890c..8a14cdc694 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1448,29 +1448,11 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) { int stat = NC_NOERR; int i,j; - char* varpath = NULL; - char* key = NULL; NCZ_FILE_INFO_T* zinfo = NULL; - NC_VAR_INFO_T* var = NULL; - NCZ_VAR_INFO_T* zvar = NULL; NCZMAP* map = NULL; - NCjson* jvar = NULL; - NCjson* jncvar = NULL; - NCjson* jdimrefs = NULL; - NCjson* jvalue = NULL; int purezarr = 0; int xarray = 0; int formatv1 = 0; - nc_type vtype; - int vtypelen; - size64_t* shapes = NULL; - int rank = 0; - int zarr_rank = 1; /* Need to watch out for scalars */ - NClist* dimnames = nclistnew(); -#ifdef ENABLE_NCZARR_FILTERS - NCjson* jfilter = NULL; - int chainindex; -#endif ZTRACE(3,"file=%s grp=%s |varnames|=%u",file->controller->path,grp->hdr.name,nclistlength(varnames)); @@ -1483,7 +1465,32 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* Load each var in turn */ for(i = 0; i < nclistlength(varnames); i++) { - const char* varname = nclistget(varnames,i); + /* per-variable info */ + NC_VAR_INFO_T* var = NULL; + NCZ_VAR_INFO_T* zvar = NULL; + NCjson* jvar = NULL; + NCjson* jncvar = NULL; + NCjson* jdimrefs = NULL; + NCjson* jvalue = NULL; + char* varpath = NULL; + char* key = NULL; + const char* varname = NULL; + size64_t* shapes = NULL; + NClist* dimnames = NULL; + int varsized = 0; + int suppress = 0; /* Abort processing of this variable */ + nc_type vtype = NC_NAT; + int vtypelen = 0; + int rank = 0; + int zarr_rank = 0; /* Need to watch out for scalars */ +#ifdef ENABLE_NCZARR_FILTERS + NCjson* jfilter = NULL; + int chainindex = 0; +#endif + + dimnames = nclistnew(); + varname = nclistget(varnames,i); + if((stat = nc4_var_list_add2(grp, varname, &var))) goto done; @@ -1522,6 +1529,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if(version != zinfo->zarr.zarr_version) {stat = (THROW(NC_ENCZARR)); goto done;} } + /* Set the type and endianness of the variable */ { int endianness; @@ -1609,23 +1617,6 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) jdimrefs = NULL; } - /* shape */ - { - if((stat = NCJdictget(jvar,"shape",&jvalue))) goto done; - if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} - if(zvar->scalar) { - rank = 0; - zarr_rank = 1; /* Zarr does not support scalars */ - } else - rank = (zarr_rank = NCJlength(jvalue)); - /* Save the rank of the variable */ - if((stat = nc4_var_set_ndims(var, rank))) goto done; - /* extract the shapes */ - if((shapes = (size64_t*)malloc(sizeof(size64_t)*zarr_rank)) == NULL) - {stat = (THROW(NC_ENOMEM)); goto done;} - if((stat = decodeints(jvalue, shapes))) goto done; - } - /* Capture dimension_separator (must precede chunk cache creation) */ { NCglobalstate* ngs = NC_getglobalstate(); @@ -1661,6 +1652,36 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) } } + /* shape */ + { + if((stat = NCJdictget(jvar,"shape",&jvalue))) goto done; + if(NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} + + /* Process the rank */ + zarr_rank = NCJlength(jvalue); + if(zarr_rank == 0) { + /* suppress variable */ + ZLOG(NCLOGWARN,"Empty shape for variable %s suppressed",var->hdr.name); + suppress = 1; + goto suppressvar; + } + + if(zvar->scalar) { + rank = 0; + zarr_rank = 1; /* Zarr does not support scalars */ + } else + rank = (zarr_rank = NCJlength(jvalue)); + + if(zarr_rank > 0) { + /* Save the rank of the variable */ + if((stat = nc4_var_set_ndims(var, rank))) goto done; + /* extract the shapes */ + if((shapes = (size64_t*)malloc(sizeof(size64_t)*zarr_rank)) == NULL) + {stat = (THROW(NC_ENOMEM)); goto done;} + if((stat = decodeints(jvalue, shapes))) goto done; + } + } + /* chunks */ { size64_t chunks[NC_MAX_VAR_DIMS]; @@ -1668,8 +1689,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if(jvalue != NULL && NCJsort(jvalue) != NCJ_ARRAY) {stat = (THROW(NC_ENCZARR)); goto done;} /* Verify the rank */ - assert (zarr_rank == NCJlength(jvalue)); - if(zvar->scalar) { + if(zvar->scalar || zarr_rank == 0) { if(var->ndims != 0) {stat = (THROW(NC_ENCZARR)); goto done;} zvar->chunkproduct = 1; @@ -1746,37 +1766,47 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; } } + /* Suppress variable if there are filters and var is not fixed-size */ + if(varsized && nclistlength((NClist*)var->filters) > 0) + suppress = 1; #endif - if((stat = computedimrefs(file, var, purezarr, xarray, rank, dimnames, shapes, var->dim))) - goto done; - - if(!zvar->scalar) { - /* Extract the dimids */ - for(j=0;jdimids[j] = var->dim[j]->hdr.id; + if(zarr_rank > 0) { + if((stat = computedimrefs(file, var, purezarr, xarray, rank, dimnames, shapes, var->dim))) + goto done; + if(!zvar->scalar) { + /* Extract the dimids */ + for(j=0;jdimids[j] = var->dim[j]->hdr.id; + } } #ifdef ENABLE_NCZARR_FILTERS - /* At this point, we can finalize the filters */ - if((stat = NCZ_filter_setup(var))) goto done; + if(!suppress) { + /* At this point, we can finalize the filters */ + if((stat = NCZ_filter_setup(var))) goto done; + } #endif +suppressvar: + if(suppress) { + /* Reclaim NCZarr variable specific info */ + (void)NCZ_zclose_var1(var); + /* Remove from list of variables and reclaim the top level var object */ + (void)nc4_var_list_del(grp, var); + var = NULL; + } + /* Clean up from last cycle */ - nclistfreeall(dimnames); dimnames = nclistnew(); + nclistfreeall(dimnames); dimnames = NULL; nullfree(varpath); varpath = NULL; nullfree(shapes); shapes = NULL; + nullfree(key); key = NULL; if(formatv1) {NCJreclaim(jncvar); jncvar = NULL;} NCJreclaim(jvar); jvar = NULL; var = NULL; } done: - nullfree(shapes); - nullfree(varpath); - nullfree(key); - nclistfreeall(dimnames); - NCJreclaim(jvar); - if(formatv1) NCJreclaim(jncvar); return ZUNTRACE(THROW(stat)); } diff --git a/nc_test4/tst_bloscfail.sh b/nc_test4/tst_bloscfail.sh index a74a3f63e5..14dfb42d1c 100755 --- a/nc_test4/tst_bloscfail.sh +++ b/nc_test4/tst_bloscfail.sh @@ -60,7 +60,6 @@ ${NCCOPY} -4 -V three_dmn_rec_var -F *,32001,0,0,0,0,1,1,0 ./tmp_bloscx3.nc ./tm # This should fail because shuffle is off if ${NCCOPY} -4 -V three_dmn_rec_var -F *,32001,0,0,0,0,1,0,0 ./tmp_bloscx3.nc ./tmp_bloscx4_fail.nc ; then echo "*** not xfail: nccopy " - exit 1; else echo "*** xfail: nccopy " fi diff --git a/ncdap_test/tst_encode.sh b/ncdap_test/tst_encode.sh index e0de50b77a..54e0a20bed 100755 --- a/ncdap_test/tst_encode.sh +++ b/ncdap_test/tst_encode.sh @@ -8,6 +8,5 @@ set -e echo "" echo "*** Testing #encode=" mechanism -#${NCDUMP} -h 'http://opendap2.oceanbrowser.net/thredds/dodsC/data/emodnet1-domains/tmp%20test.nc?lon[0:8]#encode=none' # raw: http://iridl.ldeo.columbia.edu/SOURCES/.Indices/.soi/.c8110/.anomaly/T/(Jan 1979)/VALUE/dods ${NCDUMP} -h 'http://iridl.ldeo.columbia.edu/SOURCES/.Indices/.soi/.c8110/.anomaly/T/%28Jan%201979%29/VALUE/dods?anomaly[0]' diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index 09b013823b..c38b86fe81 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -193,6 +193,8 @@ IF(ENABLE_TESTS) if(ENABLE_NCZARR_ZIP) add_sh_test(nczarr_test run_newformat) + # Test various corrupted files + ADD_SH_TEST(nczarr_test run_corrupt.sh) endif() IF(FALSE) # Obsolete tests diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index a42248da01..d8cb5d4a54 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -161,6 +161,9 @@ endif # ISMINGW endif #ENABLE_FILTER_TESTING endif #ENABLE_NCZARR_FILTERS +# Test various corrupted files +TESTS += run_corrupt.sh + endif #BUILD_UTILITIES # These programs are used by the test cases @@ -200,7 +203,8 @@ run_purezarr.sh run_interop.sh run_misc.sh \ run_filter.sh \ run_newformat.sh run_nczarr_fill.sh run_quantize.sh \ run_jsonconvention.sh run_nczfilter.sh run_unknown.sh \ -run_scalar.sh run_strings.sh run_nulls.sh run_notzarr.sh run_external.sh run_unlim_io.sh +run_scalar.sh run_strings.sh run_nulls.sh run_notzarr.sh run_external.sh \ +run_unlim_io.sh run_corrupt.sh EXTRA_DIST += \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ @@ -228,6 +232,9 @@ ref_nulls_nczarr.baseline ref_nulls_zarr.baseline ref_nulls.cdl ref_notzarr.tar. EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl \ ref_zarr_test_data.cdl.gz ref_zarr_test_data_2d.cdl.gz +# Additional Files +EXTRA_DIST += ref_noshape.file.zip + CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip BUILT_SOURCES = test_quantize.c test_filter_vlen.c test_unlim_vars.c test_endians.c \ diff --git a/nczarr_test/ref_noshape.file.zip b/nczarr_test/ref_noshape.file.zip new file mode 100644 index 0000000000000000000000000000000000000000..6f4d9c1503515f9e85a5e71c54b05a9d8fedb81b GIT binary patch literal 1481 zcmWIWW@h1H0D;PM$8azMN(eH@FchVx#pmT0XCxM+>ZN7or0R!;a56A2Jt&#F2Z&26 zxEUB(UNAE-fQbN{MhRmxs*IpvB0!gLfZb*LgQryp$ddqK5$uK;z?}wi9-7mV6M%pv zNlhV*ElErvjWvk@! zq?lYkbq57?JG`2EaMkBx?yjKj?$3@+l}jxx1#DGUx*iQ^jM4+U8xdcQVJtfOKsSRh zZg-cI6cyuhcePTyTV_s9SYl3Ts*;t0uAz>CQoLhOkYi-Li>I%vU$CdYUocQ2S}Daq zDYh0ho_BeecY>k@5BkXlrdnOY3AMX4;cs5mn}&(=sMFSR5&CC%1E&r;7=Hzlc&D_xV1G~xa?1ayJ zL`b71bY$~40Ik8|a(Id$&V1BVjBLI!6BhH~X_`3GQPVH7>C>^B4o%ItOlM^S#Tz>i NwgL6)0t;XU1^_1~a$o=e literal 0 HcmV?d00001 diff --git a/nczarr_test/run_corrupt.sh b/nczarr_test/run_corrupt.sh new file mode 100755 index 0000000000..e9ddc67d57 --- /dev/null +++ b/nczarr_test/run_corrupt.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +# Test various kinds of corrupted files + + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "$srcdir/test_nczarr.sh" + +set -e + +s3isolate "testdir_corrupt" +THISDIR=`pwd` +cd $ISOPATH + +export NCLOGGING=WARN + +testnoshape1() { + zext=file + unzip ${srcdir}/ref_noshape.file.zip + fileargs ${ISOPATH}/ref_noshape "mode=zarr,$zext" + rm -f tmp_noshape1_${zext}.cdl + ${NCDUMP} $flags $fileurl > tmp_noshape1_${zext}.cdl +} + +testnoshape2() { + # Test against the original issue URL + rm -f tmp_noshape2_gs.cdl + fileurl="https://storage.googleapis.com/cmip6/CMIP6/CMIP/NASA-GISS/GISS-E2-1-G/historical/r1i1p1f1/day/tasmin/gn/v20181015/#mode=zarr,s3&aws.profile=no" + ${NCDUMP} -h $flags $fileurl > tmp_noshape2_gs.cdl +} + +testnoshape1 +if test "x$FEATURE_S3TESTS" = xyes ; then testnoshape2; fi diff --git a/nczarr_test/run_scalar.sh b/nczarr_test/run_scalar.sh index c6de0ebc81..b7c268ee5b 100755 --- a/nczarr_test/run_scalar.sh +++ b/nczarr_test/run_scalar.sh @@ -7,7 +7,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e -s3isolate "testdir_nczarr" +s3isolate "testdir_scalar" THISDIR=`pwd` cd $ISOPATH @@ -43,7 +43,7 @@ echo "*** create nczarr file" ${NCGEN} -4 -b -o "$nczarrurl" $top_srcdir/nczarr_test/ref_scalar.cdl echo "*** read purezarr" -${NCDUMP} -n ref_scalar $zarrurl > tmp_scalar_zarr0_${zext}.cdl +${NCDUMP} -n ref_scalar $zarrurl > tmp_scalar_zarr_${zext}.cdl ${ZMD} -h $zarrurl > tmp_scalar_zarr_${zext}.txt echo "*** read nczarr" ${NCDUMP} -n ref_scalar $nczarrurl > tmp_scalar_nczarr_${zext}.cdl @@ -53,8 +53,8 @@ echo "*** verify" diff -bw $top_srcdir/nczarr_test/ref_scalar.cdl tmp_scalar_nczarr_${zext}.cdl # Fixup -zarrscalar tmp_scalar_zarr0_${zext}.cdl tmp_scalar_zarr_${zext}.cdl -diff -bw $top_srcdir/nczarr_test/ref_scalar.cdl tmp_scalar_zarr_${zext}.cdl +zarrscalar tmp_scalar_zarr_${zext}.cdl tmp_rescale_zarr_${zext}.cdl +diff -bw $top_srcdir/nczarr_test/ref_scalar.cdl tmp_rescale_zarr_${zext}.cdl } testcase file diff --git a/plugins/H5Zmisc.c b/plugins/H5Zmisc.c index 56e64734ff..8813af59d8 100644 --- a/plugins/H5Zmisc.c +++ b/plugins/H5Zmisc.c @@ -113,7 +113,7 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, break; case TC_ODDSIZE: /* Print out the chunk size */ - fprintf(stderr,"nbytes = %lld chunk size = %lld\n",(long long)nbytes,(long long)*buf_size); + fprintf(stderr,">>> nbytes = %lld chunk size = %lld\n",(long long)nbytes,(long long)*buf_size); fflush(stderr); break; default: break; @@ -122,13 +122,15 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, if (flags & H5Z_FLAG_REVERSE) { /* Decompress */ if(testcase == TC_EXPANDED) { +#ifdef DEBUG int i; float* b = (float*)*buf; -fprintf(stderr,"TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(unsigned)nbytes,(unsigned)*buf_size); +fprintf(stderr,">>> TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(unsigned)nbytes,(unsigned)*buf_size); for(i=0;i<8;i++) { fprintf(stderr," %u",(int)(b[1024+i])); } fprintf(stderr,"|\n"); +#endif /* Replace buffer */ newbuf = H5allocate_memory(*buf_size,0); if(newbuf == NULL) abort(); @@ -149,8 +151,8 @@ fprintf(stderr,"TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(u if(testcase == TC_EXPANDED) { int i; float* b; -#if 0 -fprintf(stderr,"TC_EXPANDED: compress: nbytes=%u buf_size=%u size=%u\n",(unsigned)nbytes,(unsigned)*buf_size,(unsigned)size); +#ifdef DEBUG +fprintf(stderr,">>> TC_EXPANDED: compress: nbytes=%u buf_size=%u size=%u\n",(unsigned)nbytes,(unsigned)*buf_size,(unsigned)size); #endif /* Replace buffer with one that is bigger than the input size */ newbuf = H5allocate_memory(size,0); @@ -218,7 +220,7 @@ extract1(void* field, size_t size, const unsigned int* params) llp = (unsigned long long*)field; *llp = u.ll; break; - default: fprintf(stderr,"insert: unexpected size: %u\n",(unsigned)size); abort(); + default: fprintf(stderr,">>> insert: unexpected size: %u\n",(unsigned)size); abort(); } } @@ -247,7 +249,7 @@ paramcheck(size_t nparams, const unsigned int* params, struct All* extracted) memset(&all,0,sizeof(all)); if(nparams != NPARAMS) { - fprintf(stderr,"Incorrect number of parameters: expected=%ld sent=%ld\n",(unsigned long)NPARAMS,(unsigned long)nparams); + fprintf(stderr,">>> Incorrect number of parameters: expected=%ld sent=%ld\n",(unsigned long)NPARAMS,(unsigned long)nparams); goto fail; } @@ -270,7 +272,7 @@ paramcheck(size_t nparams, const unsigned int* params, struct All* extracted) #ifdef DEBUG { size_t i; - fprintf(stderr,"bigendian=%d nparams=%d params=\n",bigendian,nparams); + fprintf(stderr,">>> nparams=%lu params=\n",nparams); for(i=0;i>> mismatch: %s\n",which); fflush(stderr); } diff --git a/test_common.in b/test_common.in index 8be771e19e..5f0f066db8 100644 --- a/test_common.in +++ b/test_common.in @@ -129,6 +129,9 @@ top_builddir="$TOPBUILDDIR" # Currently not used, but left as a Visual Studio placeholder. # VS=Debug +# Set when using gdb +#DL=".libs/" + # srcdir may or may not be defined, but if not, then create it if test "x$srcdir" = x ; then # we need to figure out our directory @@ -169,11 +172,11 @@ fi # We need to locate certain executables (and other things), # capture absolute paths, and make visible -export NCDUMP="${top_builddir}/ncdump${VS}/ncdump${ext}" -export NCCOPY="${top_builddir}/ncdump${VS}/nccopy${ext}" -export NCGEN="${top_builddir}/ncgen${VS}/ncgen${ext}" -export NCGEN3="${top_builddir}/ncgen3${VS}/ncgen3${ext}" -export NCPATHCVT="${top_builddir}/ncdump${VS}/ncpathcvt${ext}" +export NCDUMP="${top_builddir}/ncdump${VS}/${DL}ncdump${ext}" +export NCCOPY="${top_builddir}/ncdump${VS}/${DL}nccopy${ext}" +export NCGEN="${top_builddir}/ncgen${VS}/${DL}ncgen${ext}" +export NCGEN3="${top_builddir}/ncgen3${VS}/${DL}ncgen3${ext}" +export NCPATHCVT="${top_builddir}/ncdump${VS}/${DL}ncpathcvt${ext}" # Temporary hacks (until we have a test_utils directory) # to locate certain specific test files From 78d4d80139db896cb433cc859570da5a09da3b34 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 8 Oct 2023 13:47:21 -0600 Subject: [PATCH 14/77] Update release notes --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c92394242d..e15f69ab66 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,7 +6,7 @@ Release Notes {#RELEASE_NOTES} This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries. ## 4.9.3 - TBD - +* Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). From 948304a79f099508823dce55761adceb3fce1de9 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 8 Oct 2023 14:38:07 -0600 Subject: [PATCH 15/77] Fix Proxy problem for DAP2 re: Issue https://github.com/Unidata/netcdf-c/issues/2752 The authorization setup when using a proxy is apparently not being used, or used incorrectly. This PR ensures that the relevant curl options, specifically CURLOPT_VERIFYHOST and CURLOPT_VERIFYPEER, are properly setup. As part of this, the ability to turn off these options was fixed. Note that no testing of this PR is currently possible because we do not have access to a proxy. --- libdap4/d4curlfunctions.c | 65 ++++++++++++++++++++++++++------------- ncdap_test/tst_remote.sh | 4 +++ oc2/occurlfunctions.c | 64 ++++++++++++++++++++++++-------------- 3 files changed, 88 insertions(+), 45 deletions(-) diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index eb1fe9fc6d..ee06e4cacd 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -3,6 +3,11 @@ * See netcdf/COPYRIGHT file for copying and redistribution conditions. *********************************************************************/ +/* WARNING: oc2/occurlfunctions.c and libdap4/d4curlfunctions.c +should be merged since they are essentially the same file. +In the meantime, changes to one should be propagated to the other. +*/ + #include "d4includes.h" #include "d4curlfunctions.h" @@ -123,33 +128,43 @@ set_curlflag(NCD4INFO* state, int flag) } } break; - case CURLOPT_USE_SSL: - case CURLOPT_SSLCERT: case CURLOPT_SSLKEY: - case CURLOPT_SSL_VERIFYPEER: case CURLOPT_SSL_VERIFYHOST: - { - struct ssl* ssl = &state->auth->ssl; + case CURLOPT_SSL_VERIFYPEER: /* VERIFYPEER == 0 => VERIFYHOST == 0 */ /* We need to have 2 states: default and a set value */ - /* So -1 => default, >= 0 => use value; */ - if(ssl->verifypeer >= 0) - SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(ssl->verifypeer)); + /* So -1 => default >= 0 => use value */ + if(state->auth->ssl.verifypeer >= 0) { + SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(state->auth->ssl.verifypeer)); + if(state->auth->ssl.verifypeer == 0) state->auth->ssl.verifyhost = 0; + } + break; + case CURLOPT_SSL_VERIFYHOST: #ifdef HAVE_LIBCURL_766 - if(ssl->verifyhost >= 0) - SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(ssl->verifyhost)); + if(state->auth->ssl.verifyhost >= 0) { + SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(state->auth->ssl.verifyhost)); + } #endif - if(ssl->certificate) - SETCURLOPT(state, CURLOPT_SSLCERT, ssl->certificate); - if(ssl->key) - SETCURLOPT(state, CURLOPT_SSLKEY, ssl->key); - if(ssl->keypasswd) + break; + case CURLOPT_SSLCERT: + if(state->auth->ssl.certificate) + SETCURLOPT(state, CURLOPT_SSLCERT, state->auth->ssl.certificate); + break; + case CURLOPT_SSLKEY: + if(state->auth->ssl.key) + SETCURLOPT(state, CURLOPT_SSLKEY, state->auth->ssl.key); + if(state->auth->ssl.keypasswd) /* libcurl prior to 7.16.4 used 'CURLOPT_SSLKEYPASSWD' */ - SETCURLOPT(state, CURLOPT_KEYPASSWD, ssl->keypasswd); - if(ssl->cainfo) - SETCURLOPT(state, CURLOPT_CAINFO, ssl->cainfo); - if(ssl->capath) - SETCURLOPT(state, CURLOPT_CAPATH, ssl->capath); - } - break; + SETCURLOPT(state, CURLOPT_SSLKEYPASSWD, state->auth->ssl.keypasswd); + break; + case CURLOPT_CAINFO: + if(state->auth->ssl.cainfo) + SETCURLOPT(state, CURLOPT_CAINFO, state->auth->ssl.cainfo); + break; + case CURLOPT_CAPATH: + if(state->auth->ssl.capath) + SETCURLOPT(state, CURLOPT_CAPATH, state->auth->ssl.capath); + break; + case CURLOPT_USE_SSL: + break; #ifdef HAVE_CURLOPT_BUFFERSIZE case CURLOPT_BUFFERSIZE: @@ -200,6 +215,12 @@ NCD4_set_flags_perlink(NCD4INFO* state) if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_COOKIEJAR); if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_USERPWD); if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_PROXY); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSL_VERIFYPEER); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSL_VERIFYHOST); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSLCERT); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSLKEY); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_CAINFO); + if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_CAPATH); if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_USE_SSL); if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_FOLLOWLOCATION); if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_MAXREDIRS); diff --git a/ncdap_test/tst_remote.sh b/ncdap_test/tst_remote.sh index d7cc2a636f..2a4dedf72c 100755 --- a/ncdap_test/tst_remote.sh +++ b/ncdap_test/tst_remote.sh @@ -1,6 +1,10 @@ #!/bin/sh +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + if test "x$SETX" != x ; then set -x ; fi + set -e quiet=0 diff --git a/oc2/occurlfunctions.c b/oc2/occurlfunctions.c index 06b3fd352e..275d42eb4d 100644 --- a/oc2/occurlfunctions.c +++ b/oc2/occurlfunctions.c @@ -1,6 +1,11 @@ /* Copyright 2018, UCAR/Unidata and OPeNDAP, Inc. See the COPYRIGHT file for more information. */ +/* WARNING: oc2/occurlfunctions.c and libdap4/d4curlfunctions.c +should be merged since they are essentially the same file. +In the meantime, changes to one should be propagated to the other. +*/ + #include "config.h" #include #ifdef HAVE_STDINT_H @@ -127,36 +132,43 @@ ocset_curlflag(OCstate* state, int flag) } break; - case CURLOPT_USE_SSL: - case CURLOPT_SSLCERT: case CURLOPT_SSLKEY: - case CURLOPT_SSL_VERIFYPEER: case CURLOPT_SSL_VERIFYHOST: - case CURLOPT_CAINFO: case CURLOPT_CAPATH: - { - struct ssl* ssl = &state->auth->ssl; + case CURLOPT_SSL_VERIFYPEER: /* VERIFYPEER == 0 => VERIFYHOST == 0 */ /* We need to have 2 states: default and a set value */ /* So -1 => default >= 0 => use value */ - if(ssl->verifypeer >= 0) { - SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(ssl->verifypeer)); - } + if(state->auth->ssl.verifypeer >= 0) { + SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(state->auth->ssl.verifypeer)); + if(state->auth->ssl.verifypeer == 0) state->auth->ssl.verifyhost = 0; + } + break; + case CURLOPT_SSL_VERIFYHOST: #ifdef HAVE_LIBCURL_766 - if(ssl->verifyhost >= 0) { - SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(ssl->verifyhost)); + if(state->auth->ssl.verifyhost >= 0) { + SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(state->auth->ssl.verifyhost)); } #endif - if(ssl->certificate) - SETCURLOPT(state, CURLOPT_SSLCERT, ssl->certificate); - if(ssl->key) - SETCURLOPT(state, CURLOPT_SSLKEY, ssl->key); - if(ssl->keypasswd) + break; + case CURLOPT_SSLCERT: + if(state->auth->ssl.certificate) + SETCURLOPT(state, CURLOPT_SSLCERT, state->auth->ssl.certificate); + break; + case CURLOPT_SSLKEY: + if(state->auth->ssl.key) + SETCURLOPT(state, CURLOPT_SSLKEY, state->auth->ssl.key); + if(state->auth->ssl.keypasswd) /* libcurl prior to 7.16.4 used 'CURLOPT_SSLKEYPASSWD' */ - SETCURLOPT(state, CURLOPT_KEYPASSWD, ssl->keypasswd); - if(ssl->cainfo) - SETCURLOPT(state, CURLOPT_CAINFO, ssl->cainfo); - if(ssl->capath) - SETCURLOPT(state, CURLOPT_CAPATH, ssl->capath); - } - break; + SETCURLOPT(state, CURLOPT_SSLKEYPASSWD, state->auth->ssl.keypasswd); + break; + case CURLOPT_CAINFO: + if(state->auth->ssl.cainfo) + SETCURLOPT(state, CURLOPT_CAINFO, state->auth->ssl.cainfo); + break; + case CURLOPT_CAPATH: + if(state->auth->ssl.capath) + SETCURLOPT(state, CURLOPT_CAPATH, state->auth->ssl.capath); + break; + case CURLOPT_USE_SSL: + break; #ifdef HAVE_CURLOPT_BUFFERSIZE case CURLOPT_BUFFERSIZE: @@ -210,6 +222,12 @@ ocset_flags_perlink(OCstate* state) if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_COOKIEJAR); if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_USERPWD); if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_PROXY); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_SSL_VERIFYPEER); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_SSL_VERIFYHOST); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_SSLCERT); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_SSLKEY); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_CAINFO); + if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_CAPATH); if(stat == OC_NOERR) stat = ocset_curlflag(state,CURLOPT_USE_SSL); if(stat == OC_NOERR) stat = ocset_curlflag(state, CURLOPT_FOLLOWLOCATION); if(stat == OC_NOERR) stat = ocset_curlflag(state, CURLOPT_MAXREDIRS); From 23f3c8f55a891c6a4cbe1457db3744fa93f0ae94 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 8 Oct 2023 15:17:45 -0600 Subject: [PATCH 16/77] update release notes --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c92394242d..1575118a34 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). From 5fa2defc7e67c5b47a3e4ee3fbb883790ed15593 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 8 Oct 2023 19:59:28 -0600 Subject: [PATCH 17/77] Improve fetch performance of DAP4 Prior to this PR, DAP4 always fetched the whole (constrained) dataset This PR changes the query processing so 1. It reads data on a per-variable request (equivalent to calling nc_get_var()). 2. It tracks a response for every query. Most of the changes reflect having to do per-variable requests. In any case, doing all this significantly reduces the amount of data transmitted and hence speeds up DAP4 requests. --- RELEASE_NOTES.md | 1 + dap4_test/test_common.h | 18 +- dap4_test/test_meta.c | 2 +- dap4_test/test_parse.c | 2 +- include/ncuri.h | 26 +-- libdap4/d4chunk.c | 135 +++++++-------- libdap4/d4curlfunctions.c | 4 +- libdap4/d4data.c | 96 +++++++---- libdap4/d4debug.c | 2 +- libdap4/d4file.c | 348 +++++++++++++++++++++++++------------- libdap4/d4fix.c | 4 +- libdap4/d4http.c | 75 -------- libdap4/d4meta.c | 110 ++++-------- libdap4/d4parser.c | 34 ++-- libdap4/d4read.c | 121 +++---------- libdap4/d4read.h | 2 +- libdap4/d4swap.c | 70 ++++---- libdap4/d4util.c | 69 ++++---- libdap4/d4util.h | 5 +- libdap4/d4varx.c | 209 +++++++++++++++++------ libdap4/ncd4.h | 40 +++-- libdap4/ncd4dispatch.c | 11 +- libdap4/ncd4types.h | 96 ++++++----- libdispatch/dinfermodel.c | 97 +++++------ libdispatch/ncuri.c | 285 ++++++++++++++++++------------- libnczarr/zarr.c | 27 +-- libnczarr/zarr.h | 4 +- libnczarr/zclose.c | 2 +- libnczarr/zcreate.c | 2 +- libnczarr/zinternal.h | 2 +- libnczarr/zopen.c | 2 +- 31 files changed, 1010 insertions(+), 891 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c92394242d..ed19b87f71 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Improve the speed and data quantity for DAP4 queries. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). diff --git a/dap4_test/test_common.h b/dap4_test/test_common.h index a005d9c9cd..d7e096e2da 100644 --- a/dap4_test/test_common.h +++ b/dap4_test/test_common.h @@ -21,6 +21,7 @@ typedef int TDMR; static NCbytes* input = NULL; static NCbytes* output = NULL; static NCD4meta* metadata = NULL; +static NCD4response* resp = NULL; static char* infile = NULL; static char* outfile = NULL; static int ncid = 0; @@ -85,16 +86,21 @@ setup(int tdmr, int argc, char** argv) if(translatenc4) controller->controls.translation = NCD4_TRANSNC4; NCD4_applyclientfragmentcontrols(controller); - if((metadata=NCD4_newmeta(controller))==NULL) - fail(NC_ENOMEM); - metadata->mode = mode; - NCD4_attachraw(metadata, ncbyteslength(input),ncbytescontents(input)); - if((ret=NCD4_dechunk(metadata))) /* ok for mode == DMR or mode == DAP */ + if((ret=NCD4_newMeta(controller,&metadata))) + fail(ret); + + if((ret=NCD4_newResponse(controller,&resp))) + fail(ret); + resp->raw.size = ncbyteslength(input); + resp->raw.memory = ncbytescontents(input); + resp->mode = mode; + + if((ret=NCD4_dechunk(resp))) /* ok for mode == DMR or mode == DAP */ fail(ret); #ifdef DEBUG { - int swap = (metadata->serial.hostbigendian != metadata->serial.remotebigendian); + int swap = (controller->platform.hostlittleendian != resp->remotelittleendian); void* d = metadata->serial.dap; size_t sz = metadata->serial.dapsize; fprintf(stderr,"====================\n"); diff --git a/dap4_test/test_meta.c b/dap4_test/test_meta.c index f0bf2273b4..478fcfb62d 100644 --- a/dap4_test/test_meta.c +++ b/dap4_test/test_meta.c @@ -20,7 +20,7 @@ main(int argc, char** argv) fprintf(stderr,"t_dmrmeta %s -> %s\n",infile,outfile); #endif - if((ret = NCD4_parse(metadata))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; if((ret = NCD4_metabuild(metadata,ncid))) goto done; done: diff --git a/dap4_test/test_parse.c b/dap4_test/test_parse.c index 44075ffaa8..59afb74d5e 100644 --- a/dap4_test/test_parse.c +++ b/dap4_test/test_parse.c @@ -17,7 +17,7 @@ main(int argc, char** argv) setup(TDMR_PARSE,argc,argv); - if((ret = NCD4_parse(metadata))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; ret = NCD4_print(metadata,output); ncbytesnull(output); if(ret == NC_NOERR) { diff --git a/include/ncuri.h b/include/ncuri.h index eafb9be97f..bd1b641012 100644 --- a/include/ncuri.h +++ b/include/ncuri.h @@ -35,12 +35,8 @@ typedef struct NCURI { char* path; /*!< path */ char* query; /*!< query */ char* fragment; /*!< fragment */ - char** fraglist; /* envv style list of decomposed fragment*/ - char** querylist; /* envv style list of decomposed query*/ -#if 0 - char* projection; /*!< without leading '?'*/ - char* selection; /*!< with leading '&'*/ -#endif + void* fraglist; /* some representation of the decomposed fragment string */ + void* querylist; /* some representation of the decomposed query string */ } NCURI; #if 0 @@ -90,6 +86,18 @@ EXTERNL int ncurisetfragmentkey(NCURI* duri,const char* key, const char* value); /* append a specific &key=...& in uri fragment */ EXTERNL int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value); +/* Replace a specific &key=...& in uri query */ +EXTERNL int ncurisetquerykey(NCURI* duri,const char* key, const char* value); + +/* append a specific &key=...& in uri query */ +EXTERNL int ncuriappendquerykey(NCURI* duri,const char* key, const char* value); + +/* Get the actual list of queryies */ +EXTERNL void* ncuriqueryparams(NCURI* uri); +/* Get the actual list of frags */ +EXTERNL void* ncurifragmentparams(NCURI* uri); + + /* Construct a complete NC URI; caller frees returned string */ EXTERNL char* ncuribuild(NCURI*,const char* prefix, const char* suffix, int flags); @@ -105,12 +113,6 @@ EXTERNL const char* ncurifragmentlookup(NCURI*, const char* param); */ EXTERNL const char* ncuriquerylookup(NCURI*, const char* param); -/* Obtain the complete list of fragment pairs in envv format */ -EXTERNL const char** ncurifragmentparams(NCURI*); - -/* Obtain the complete list of query pairs in envv format */ -EXTERNL const char** ncuriqueryparams(NCURI*); - /* URL Encode/Decode */ EXTERNL char* ncuridecode(const char* s); /* Partial decode */ diff --git a/libdap4/d4chunk.c b/libdap4/d4chunk.c index e95f34b9c9..3f2fe44d7d 100644 --- a/libdap4/d4chunk.c +++ b/libdap4/d4chunk.c @@ -22,133 +22,128 @@ and whether it has checksums. */ /* Forward */ -static int processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count); +static int processerrchunk(NCD4response*, void* errchunk, unsigned int count); /**************************************************/ - -void -NCD4_resetSerial(NCD4serial* serial, size_t rawsize, void* rawdata) -{ - nullfree(serial->errdata); - nullfree(serial->dmr); - nullfree(serial->dap); - nullfree(serial->rawdata); - /* clear all fields */ - memset(serial,0,sizeof(NCD4serial)); - /* Reset fields */ - serial->hostlittleendian = NCD4_isLittleEndian(); - serial->rawsize = rawsize; - serial->rawdata = rawdata; -} - int -NCD4_dechunk(NCD4meta* metadata) +NCD4_dechunk(NCD4response* resp) { - unsigned char *praw, *phdr, *pdap; + unsigned char *praw, *pdmr, *phdr, *pdap, *pappend, *pchunk; NCD4HDR hdr; + int firstchunk; #ifdef D4DUMPRAW - NCD4_tagdump(metadata->serial.rawsize,metadata->serial.rawdata,0,"RAW"); + NCD4_tagdump(resp->serial.raw.size,resp->serial.raw.data,0,"RAW"); #endif /* Access the returned raw data */ - praw = metadata->serial.rawdata; + praw = (unsigned char*)resp->raw.memory; - if(metadata->mode == NCD4_DSR) { + if(resp->mode == NCD4_DSR) { return THROW(NC_EDMR); - } else if(metadata->mode == NCD4_DMR) { + } else if(resp->mode == NCD4_DMR) { /* Verify the mode; assume that the is optional */ if(memcmp(praw,"serial.rawsize; - if((metadata->serial.dmr = malloc(len+1)) == NULL) + len = resp->raw.size; + if((resp->serial.dmr = malloc(len+1)) == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.dmr,praw,len); - metadata->serial.dmr[len] = '\0'; + memcpy(resp->serial.dmr,praw,len); + resp->serial.dmr[len] = '\0'; /* Suppress nuls */ - (void)NCD4_elidenuls(metadata->serial.dmr,len); + (void)NCD4_elidenuls(resp->serial.dmr,len); return THROW(NC_NOERR); } - } else if(metadata->mode != NCD4_DAP) + } else if(resp->mode != NCD4_DAP) return THROW(NC_EDAP); /* We must be processing a DAP mode packet */ - praw = (metadata->serial.dap = metadata->serial.rawdata); - metadata->serial.rawdata = NULL; + praw = resp->raw.memory; /* If the raw data looks like xml, then we almost certainly have an error */ if(memcmp(praw,"serial.rawsize, metadata->serial.rawdata); + int stat = NCD4_seterrormessage(resp, resp->raw.size, resp->raw.memory); return THROW(stat); /* slight lie */ } - /* Get the DMR chunk header*/ - phdr = NCD4_getheader(praw,&hdr,metadata->serial.hostlittleendian); + /* Get the first header to get dmr content and endian flags*/ + pdmr = NCD4_getheader(praw,&hdr,resp->controller->platform.hostlittleendian); if(hdr.count == 0) return THROW(NC_EDMR); - if(hdr.flags & NCD4_ERR_CHUNK) { - return processerrchunk(metadata, (void*)phdr, hdr.count); - } + if(hdr.flags & NCD4_ERR_CHUNK) + return processerrchunk(resp, (void*)pdmr, hdr.count); + resp->remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0); - metadata->serial.remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0); - /* Again, avoid strxxx operations on dmr */ - if((metadata->serial.dmr = malloc(hdr.count+1)) == NULL) + /* avoid strxxx operations on dmr */ + if((resp->serial.dmr = malloc(hdr.count+1)) == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.dmr,phdr,hdr.count); - metadata->serial.dmr[hdr.count-1] = '\0'; + memcpy(resp->serial.dmr,pdmr,hdr.count); + resp->serial.dmr[hdr.count-1] = '\0'; /* Suppress nuls */ - (void)NCD4_elidenuls(metadata->serial.dmr,hdr.count); + (void)NCD4_elidenuls(resp->serial.dmr,hdr.count); + /* See if there is any data after the DMR */ if(hdr.flags & NCD4_LAST_CHUNK) return THROW(NC_ENODATA); /* Read and concat together the data chunks */ - phdr = phdr + hdr.count; /* point to data chunk header */ + phdr = pdmr + hdr.count; /* point to data chunk header */ /* Do a sanity check in case the server has shorted us with no data */ - if((hdr.count + CHUNKHDRSIZE) >= metadata->serial.rawsize) { + if((hdr.count + CHUNKHDRSIZE) >= resp->raw.size) { /* Server only sent the DMR part */ - metadata->serial.dapsize = 0; + resp->serial.dapsize = 0; return THROW(NC_EDATADDS); } - pdap = metadata->serial.dap; - for(;;) { - phdr = NCD4_getheader(phdr,&hdr,metadata->serial.hostlittleendian); - if(hdr.flags & NCD4_ERR_CHUNK) { - return processerrchunk(metadata, (void*)phdr, hdr.count); + /* walk all the data chunks */ + /* invariants: + praw -- beginning of the raw response + pdmr -- beginning of the dmr in the raw data + pdap -- beginning of the dechunked dap data + phdr -- pointer to the hdr of the current chunk + pchunk -- pointer to the data part of the current chunk + pappend -- where to append next chunk to the growing dechunked data + */ + for(firstchunk=1;;firstchunk=0) { + pchunk = NCD4_getheader(phdr,&hdr,resp->controller->platform.hostlittleendian); /* Process first data chunk header */ + if(firstchunk) { + pdap = phdr; /* remember start point of the dechunked data */ + pappend = phdr; /* start appending here */ } + if(hdr.flags & NCD4_ERR_CHUNK) + return processerrchunk(resp, (void*)pchunk, hdr.count); /* data chunk; possibly last; possibly empty */ - if(hdr.count > 0) { - d4memmove(pdap,phdr,hdr.count); /* will overwrite the header */ - phdr += hdr.count; - pdap += hdr.count; - } + if(hdr.count > 0) + d4memmove(pappend,pchunk,hdr.count); /* overwrite the header; this the heart of dechunking */ + pappend += hdr.count; /* next append point */ + phdr = pchunk + hdr.count; /* point to header of next chunk */ if(hdr.flags & NCD4_LAST_CHUNK) break; } - metadata->serial.dapsize = (size_t)DELTA(pdap,metadata->serial.dap); + resp->serial.dap = pdap; /* start of dechunked data */ + resp->serial.dapsize = (size_t)DELTA(pappend,pdap); #ifdef D4DUMPDMR - fprintf(stderr,"%s\n",metadata->serial.dmr); + fprintf(stderr,"%s\n",resp->serial.dmr); fflush(stderr); #endif #ifdef D4DUMPDAP - NCD4_tagdump(metadata->serial.dapsize,metadata->serial.dap,0,"DAP"); + NCD4_tagdump(resp->serial.dapsize,resp->serial.dap,0,"DAP"); #endif return THROW(NC_NOERR); } static int -processerrchunk(NCD4meta* metadata, void* errchunk, unsigned int count) +processerrchunk(NCD4response* resp, void* errchunk, unsigned int count) { - metadata->serial.errdata = (char*)d4alloc(count+1); - if(metadata->serial.errdata == NULL) + resp->serial.errdata = (char*)d4alloc(count+1); + if(resp->serial.errdata == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->serial.errdata,errchunk,count); - metadata->serial.errdata[count] = '\0'; + memcpy(resp->serial.errdata,errchunk,count); + resp->serial.errdata[count] = '\0'; return THROW(NC_ENODATA); /* slight lie */ } @@ -157,26 +152,26 @@ Given a raw response, attempt to infer the mode: DMR, DAP, DSR. Since DSR is not standardizes, it becomes the default. */ int -NCD4_infermode(NCD4meta* meta) +NCD4_infermode(NCD4response* resp) { - d4size_t size = meta->serial.rawsize; - char* raw = meta->serial.rawdata; + d4size_t size = resp->raw.size; + char* raw = resp->raw.memory; if(size < 16) return THROW(NC_EDAP); /* must have at least this to hold a hdr + partial dmr*/ if(memcmp(raw,"mode = NCD4_DMR; + resp->mode = NCD4_DMR; goto done; } raw += 4; /* Pretend we have a DAP hdr */ if(memcmp(raw,"mode = NCD4_DAP; + resp->mode = NCD4_DAP; goto done; } /* Default to DSR */ - meta->mode = NCD4_DSR; + resp->mode = NCD4_DSR; done: return NC_NOERR; diff --git a/libdap4/d4curlfunctions.c b/libdap4/d4curlfunctions.c index eb1fe9fc6d..e60a05673f 100644 --- a/libdap4/d4curlfunctions.c +++ b/libdap4/d4curlfunctions.c @@ -319,7 +319,7 @@ NCD4_get_rcproperties(NCD4INFO* state) ncerror err = NC_NOERR; char* option = NULL; #ifdef HAVE_CURLOPT_BUFFERSIZE - option = NC_rclookup(D4BUFFERSIZE,state->uri->uri,NULL); + option = NC_rclookup(D4BUFFERSIZE,state->dmruri->uri,NULL); if(option != NULL && strlen(option) != 0) { long bufsize; if(strcasecmp(option,"max")==0) @@ -330,7 +330,7 @@ NCD4_get_rcproperties(NCD4INFO* state) } #endif #ifdef HAVE_CURLOPT_KEEPALIVE - option = NC_rclookup(D4KEEPALIVE,state->uri->uri,NULL); + option = NC_rclookup(D4KEEPALIVE,state->dmruri->uri,NULL); if(option != NULL && strlen(option) != 0) { /* The keepalive value is of the form 0 or n/m, where n is the idle time and m is the interval time; diff --git a/libdap4/d4data.c b/libdap4/d4data.c index 5d61b81dc4..234f3e3776 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -16,7 +16,7 @@ This code serves two purposes (NCD4_processdata) 2. Walk a specified variable instance to convert to netcdf4 memory representation. - (NCD4_fillinstance) + (NCD4_movetoinstance) */ @@ -29,7 +29,6 @@ static int fillopfixed(NCD4meta*, d4size_t opaquesize, NCD4offset* offset, void* static int fillopvar(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillstruct(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillseq(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); -static int NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel); static unsigned NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar); /***************************************************/ @@ -54,8 +53,9 @@ static unsigned int debugcrc32(unsigned int crc, const void *buf, size_t size) /***************************************************/ /* API */ +/* Parcel out the dechunked data to the corresponding vars */ int -NCD4_processdata(NCD4meta* meta) +NCD4_parcelvars(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i; @@ -68,35 +68,55 @@ NCD4_processdata(NCD4meta* meta) toplevel = nclistnew(); NCD4_getToplevelVars(meta,root,toplevel); - /* Otherwise */ - NCD4_inferChecksums(meta,toplevel); + /* Compute the offset and size of the toplevel vars in the raw dap data. */ + offset = BUILDOFFSET(resp->serial.dap,resp->serial.dapsize); + for(i=0;iinferredchecksumming))) { + FAIL(ret,"delimit failure"); + } + var->data.response = resp; /* cross link */ + } +done: + return THROW(ret); +} + +/* Process top level vars wrt checksums and swapping */ +int +NCD4_processdata(NCD4meta* meta, NCD4response* resp) +{ + int ret = NC_NOERR; + int i; + NClist* toplevel = NULL; + NCD4node* root = meta->root; + NCD4offset* offset = NULL; - /* If necessary, byte swap the serialized data */ /* Do we need to swap the dap4 data? */ - meta->swap = (meta->serial.hostlittleendian != meta->serial.remotelittleendian); + meta->swap = (meta->controller->platform.hostlittleendian != resp->remotelittleendian); - /* Compute the offset and size of the toplevel vars in the raw dap data. */ - /* Also extract remote checksums */ - offset = BUILDOFFSET(meta->serial.dap,meta->serial.dapsize); + /* Recursively walk the tree in prefix order + to get the top-level variables; also mark as unvisited */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,root,toplevel); + + /* Extract remote checksums */ for(i=0;icontroller->data.inferredchecksumming) { - /* Compute remote checksum: must occur before any byte swapping */ + if(resp->inferredchecksumming) { + /* Compute checksum of response data: must occur before any byte swapping and after delimiting */ var->data.localchecksum = NCD4_computeChecksum(meta,var); #ifdef DUMPCHECKSUM fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum); #endif /* verify checksums */ - if(!meta->controller->data.checksumignore) { + if(!resp->checksumignore) { if(var->data.localchecksum != var->data.remotechecksum) { nclog(NCLOGERR,"Checksum mismatch: %s\n",var->name); ret = NC_EDAP; goto done; } /* Also verify checksum attribute */ - if(meta->controller->data.attrchecksumming) { + if(resp->attrchecksumming) { if(var->data.attrchecksum != var->data.remotechecksum) { nclog(NCLOGERR,"Attribute Checksum mismatch: %s\n",var->name); ret = NC_EDAP; @@ -105,13 +125,11 @@ NCD4_processdata(NCD4meta* meta) } } } - } - - /* Swap the data for each top level variable, - */ - if(meta->swap) { - if((ret=NCD4_swapdata(meta,toplevel))) - FAIL(ret,"byte swapping failed"); + if(meta->swap) { + if((ret=NCD4_swapdata(resp,var,meta->swap))) + FAIL(ret,"byte swapping failed"); + } + var->data.valid = 1; /* Everything should be in place */ } done: @@ -133,7 +151,7 @@ Assumes that NCD4_processdata has been called. */ int -NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs) +NCD4_movetoinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs) { int ret = NC_NOERR; void* dst = *dstp; @@ -149,30 +167,30 @@ NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dst } else switch(type->subsort) { case NC_STRING: /* oob strings */ if((ret=fillstring(meta,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; case NC_OPAQUE: if(type->opaque.size > 0) { /* We know the size and its the same for all instances */ if((ret=fillopfixed(meta,type->opaque.size,offset,&dst))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } else { /* Size differs per instance, so we need to convert each opaque to a vlen */ if((ret=fillopvar(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } break; case NC_STRUCT: if((ret=fillstruct(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; case NC_SEQ: if((ret=fillseq(meta,type,offset,&dst,blobs))) - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); break; default: ret = NC_EINVAL; - FAIL(ret,"fillinstance"); + FAIL(ret,"movetoinstance"); } *dstp = dst; @@ -196,7 +214,7 @@ fillstruct(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NCli NCD4node* field = nclistget(type->vars,i); NCD4node* ftype = field->basetype; void* fdst = (((char*)dst) + field->meta.offset); - if((ret=NCD4_fillinstance(meta,ftype,offset,&fdst,blobs))) + if((ret=NCD4_movetoinstance(meta,ftype,offset,&fdst,blobs))) FAIL(ret,"fillstruct"); } dst = ((char*)dst) + type->meta.memsize; @@ -231,7 +249,7 @@ fillseq(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* for(i=0;ip))+(recordsize * i); - if((ret=NCD4_fillinstance(meta,vlentype,offset,&recdst,blobs))) + if((ret=NCD4_movetoinstance(meta,vlentype,offset,&recdst,blobs))) FAIL(ret,"fillseq"); } dst++; @@ -373,12 +391,16 @@ fprintf(stderr,"toplevel: var=%s\n",node->name); return THROW(ret); } -static int -NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel) +int +NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i, attrfound; - NCD4INFO* info = meta->controller; + NClist* toplevel = NULL; + + /* Get the toplevel vars */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,meta->root,toplevel); /* First, look thru the DMR to see if there is a checksum attribute */ attrfound = 0; @@ -399,9 +421,9 @@ NCD4_inferChecksums(NCD4meta* meta, NClist* toplevel) } } } - info->data.attrchecksumming = (attrfound ? 1 : 0); + resp->attrchecksumming = (attrfound ? 1 : 0); /* Infer checksums */ - info->data.inferredchecksumming = ((info->data.attrchecksumming || info->data.querychecksumming) ? 1 : 0); + resp->inferredchecksumming = ((resp->attrchecksumming || resp->querychecksumming) ? 1 : 0); return THROW(ret); } diff --git a/libdap4/d4debug.c b/libdap4/d4debug.c index d2d40b44a6..caed80027a 100644 --- a/libdap4/d4debug.c +++ b/libdap4/d4debug.c @@ -97,7 +97,7 @@ int NCD4_debugcopy(NCD4INFO* info) { int i,ret=NC_NOERR; - NCD4meta* meta = info->substrate.metadata; + NCD4meta* meta = info->dmrmetadata; NClist* topvars = nclistnew(); NC* ncp = info->controller; void* memory = NULL; diff --git a/libdap4/d4file.c b/libdap4/d4file.c index aeccc7423f..471afa15d2 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -6,7 +6,6 @@ #include "ncdispatch.h" #include "ncd4dispatch.h" #include "d4includes.h" -#include "d4read.h" #include "d4curlfunctions.h" #ifdef _MSC_VER @@ -23,19 +22,20 @@ static int constrainable(NCURI*); static void freeCurl(NCD4curl*); -static void freeInfo(NCD4INFO*); static int fragmentcheck(NCD4INFO*, const char* key, const char* subkey); static const char* getfragment(NCD4INFO* info, const char* key); static const char* getquery(NCD4INFO* info, const char* key); static int set_curl_properties(NCD4INFO*); static int makesubstrate(NCD4INFO* d4info); -static void resetInfoforRead(NCD4INFO* d4info); /**************************************************/ /* Constants */ static const char* checkseps = "+,:;"; +/*Define the set of protocols known to be constrainable */ +static const char* constrainableprotocols[] = {"http", "https",NULL}; + /**************************************************/ int NCD4_open(const char * path, int mode, @@ -49,7 +49,8 @@ NCD4_open(const char * path, int mode, NCD4meta* meta = NULL; size_t len = 0; void* contents = NULL; - + NCD4response* dmrresp = NULL; + if(path == NULL) return THROW(NC_EDAPURL); @@ -61,29 +62,27 @@ NCD4_open(const char * path, int mode, /* Setup our NC and NCDAPCOMMON state*/ - d4info = (NCD4INFO*)calloc(1,sizeof(NCD4INFO)); - if(d4info == NULL) {ret = NC_ENOMEM; goto done;} - + if((ret=NCD4_newInfo(&d4info))) goto done; nc->dispatchdata = d4info; nc->int_ncid = nc__pseudofd(); /* create a unique id */ d4info->controller = (NC*)nc; /* Parse url and params */ - if(ncuriparse(nc->path,&d4info->uri)) + if(ncuriparse(nc->path,&d4info->dmruri)) {ret = NC_EDAPURL; goto done;} /* Load auth info from rc file */ - if((ret = NC_authsetup(&d4info->auth, d4info->uri))) + if((ret = NC_authsetup(&d4info->auth, d4info->dmruri))) goto done; NCD4_curl_protocols(d4info); - if(!constrainable(d4info->uri)) + if(!constrainable(d4info->dmruri)) SETFLAG(d4info->controls.flags,NCF_UNCONSTRAINABLE); /* fail if we are unconstrainable but have constraints */ if(FLAGSET(d4info->controls.flags,NCF_UNCONSTRAINABLE)) { - if(d4info->uri != NULL) { - const char* ce = ncuriquerylookup(d4info->uri,DAP4CE); /* Look for dap4.ce */ + if(d4info->dmruri != NULL) { + const char* ce = ncuriquerylookup(d4info->dmruri,DAP4CE); /* Look for dap4.ce */ if(ce != NULL) { nclog(NCLOGWARN,"Attempt to constrain an unconstrainable data source: %s=%s", DAP4CE,ce); @@ -115,7 +114,7 @@ NCD4_open(const char * path, int mode, } /* Turn on logging; only do this after oc_open*/ - if((value = ncurifragmentlookup(d4info->uri,"log")) != NULL) { + if((value = ncurifragmentlookup(d4info->dmruri,"log")) != NULL) { ncloginit(); ncsetloglevel(NCLOGNOTE); } @@ -150,30 +149,35 @@ NCD4_open(const char * path, int mode, /* Reset the substrate */ if((ret=makesubstrate(d4info))) goto done; - /* Always start by reading the DMR only */ + /* Always start by reading the whole DMR only */ /* reclaim substrate.metadata */ - resetInfoforRead(d4info); + NCD4_resetInfoForRead(d4info); /* Rebuild metadata */ - if((d4info->substrate.metadata=NCD4_newmeta(d4info))==NULL) - {ret = NC_ENOMEM; goto done;} + if((ret = NCD4_newMeta(d4info,&d4info->dmrmetadata))) goto done; + meta = d4info->dmrmetadata; + + /* Capture response */ + if((dmrresp = (NCD4response*)calloc(1,sizeof(NCD4response)))==NULL) + {ret = NC_ENOMEM; goto done;} + dmrresp->controller = d4info; - if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags))) goto done; + if((ret=NCD4_readDMR(d4info, d4info->controls.flags.flags, d4info->dmruri, dmrresp))) goto done; /* set serial.rawdata */ len = ncbyteslength(d4info->curl->packet); contents = ncbytesextract(d4info->curl->packet); - NCD4_attachraw(d4info->substrate.metadata, len, contents); - - /* process query parameters */ - NCD4_applyclientquerycontrols(d4info); + assert(dmrresp != NULL); + dmrresp->raw.size = len; + dmrresp->raw.memory = contents; - meta = d4info->substrate.metadata; + /* process checksum parameters */ + NCD4_applychecksumcontrols(d4info,dmrresp); /* Infer the mode */ - if((ret=NCD4_infermode(meta))) goto done; + if((ret=NCD4_infermode(dmrresp))) goto done; /* Process the dmr part */ - if((ret=NCD4_dechunk(meta))) goto done; + if((ret=NCD4_dechunk(dmrresp))) goto done; #ifdef D4DUMPDMR { @@ -184,13 +188,13 @@ NCD4_open(const char * path, int mode, } #endif - if((ret = NCD4_parse(d4info->substrate.metadata))) goto done; + if((ret = NCD4_parse(meta,dmrresp,0))) goto done; #ifdef D4DEBUGMETA { fprintf(stderr,"\n/////////////\n"); NCbytes* buf = ncbytesnew(); - NCD4_print(d4info->substrate.metadata,buf); + NCD4_print(meta,buf); ncbytesnull(buf); fputs(ncbytescontents(buf),stderr); ncbytesfree(buf); @@ -200,12 +204,20 @@ NCD4_open(const char * path, int mode, #endif /* Build the substrate metadata */ - ret = NCD4_metabuild(d4info->substrate.metadata,d4info->substrate.metadata->ncid); + ret = NCD4_metabuild(meta,meta->ncid); if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done; + /* Remember the response */ + nclistpush(d4info->responses,dmrresp); + + /* Avoid duplicate reclaims */ + dmrresp = NULL; + d4info = NULL; + done: + NCD4_reclaimResponse(dmrresp); + NCD4_reclaimInfo(d4info); if(ret) { - freeInfo(d4info); nc->dispatchdata = NULL; } return THROW(ret); @@ -236,7 +248,7 @@ NCD4_close(int ncid, void* ignore) ret = nc_abort(substrateid); } - freeInfo(d4info); + NCD4_reclaimInfo(d4info); done: return THROW(ret); @@ -248,82 +260,6 @@ NCD4_abort(int ncid) return NCD4_close(ncid,NULL); } -/**************************************************/ - -/* Reclaim an NCD4INFO instance */ -static void -freeInfo(NCD4INFO* d4info) -{ - if(d4info == NULL) return; - d4info->controller = NULL; /* break link */ - nullfree(d4info->rawurltext); - nullfree(d4info->urltext); - ncurifree(d4info->uri); - freeCurl(d4info->curl); - nullfree(d4info->data.memory); - nullfree(d4info->data.ondiskfilename); - if(d4info->data.ondiskfile != NULL) - fclose(d4info->data.ondiskfile); - nullfree(d4info->fileproto.filename); - if(d4info->substrate.realfile - && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { - /* We used real file, so we need to delete the temp file - unless we are debugging. - Assume caller has done nc_close|nc_abort on the ncid. - Note that in theory, this should not be necessary since - AFAIK the substrate file is still in def mode, and - when aborted, it should be deleted. But that is not working - for some reason, so we delete it ourselves. - */ - if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); - } - } - nullfree(d4info->substrate.filename); /* always reclaim */ - NCD4_reclaimMeta(d4info->substrate.metadata); - NC_authfree(d4info->auth); - nclistfree(d4info->blobs); - free(d4info); -} - -/* Reset NCD4INFO instance for new read request */ -static void -resetInfoforRead(NCD4INFO* d4info) -{ - if(d4info == NULL) return; - if(d4info->substrate.realfile - && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { - /* We used real file, so we need to delete the temp file - unless we are debugging. - Assume caller has done nc_close|nc_abort on the ncid. - Note that in theory, this should not be necessary since - AFAIK the substrate file is still in def mode, and - when aborted, it should be deleted. But that is not working - for some reason, so we delete it ourselves. - */ - if(d4info->substrate.filename != NULL) { - unlink(d4info->substrate.filename); - } - } - NCD4_resetMeta(d4info->substrate.metadata); - nullfree(d4info->substrate.metadata); - d4info->substrate.metadata = NULL; -} - -static void -freeCurl(NCD4curl* curl) -{ - if(curl == NULL) return; - NCD4_curlclose(curl->curl); - ncbytesfree(curl->packet); - nullfree(curl->errdata.code); - nullfree(curl->errdata.message); - free(curl); -} - -/* Define the set of protocols known to be constrainable */ -static const char* constrainableprotocols[] = {"http", "https",NULL}; - static int constrainable(NCURI* durl) { @@ -449,11 +385,6 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info) if(value != NULL) strncpy(info->controls.substratename,value,(NC_MAX_NAME-1)); - value = getfragment(info,"hyrax"); - if(value != NULL) { - info->data.checksumignore = 1; /* Assume checksum, but ignore */ - } - info->controls.opaquesize = DFALTOPAQUESIZE; value = getfragment(info,"opaquesize"); if(value != NULL) { @@ -476,22 +407,29 @@ NCD4_applyclientfragmentcontrols(NCD4INFO* info) } } +/* Checksum controls are found both in the query and fragment + parts of a URL. +*/ void -NCD4_applyclientquerycontrols(NCD4INFO* info) +NCD4_applychecksumcontrols(NCD4INFO* info, NCD4response* resp) { const char* value = getquery(info,DAP4CSUM); if(value == NULL) { - info->data.querychecksumming = DEFAULT_CHECKSUM_STATE; + resp->querychecksumming = DEFAULT_CHECKSUM_STATE; } else { if(strcasecmp(value,"false")==0) { - info->data.querychecksumming = 0; + resp->querychecksumming = 0; } else if(strcasecmp(value,"true")==0) { - info->data.querychecksumming = 1; + resp->querychecksumming = 1; } else { nclog(NCLOGWARN,"Unknown checksum mode: %s ; using default",value); - info->data.querychecksumming = DEFAULT_CHECKSUM_STATE; + resp->querychecksumming = DEFAULT_CHECKSUM_STATE; } } + value = getfragment(info,"hyrax"); + if(value != NULL) { + resp->checksumignore = 1; /* Assume checksum, but ignore */ + } } /* Search for substring in value of param. If substring == NULL; then just @@ -523,7 +461,7 @@ getfragment(NCD4INFO* info, const char* key) const char* value; if(info == NULL || key == NULL) return NULL; - if((value=ncurifragmentlookup(info->uri,key)) == NULL) + if((value=ncurifragmentlookup(info->dmruri,key)) == NULL) return NULL; return value; } @@ -537,7 +475,7 @@ getquery(NCD4INFO* info, const char* key) const char* value; if(info == NULL || key == NULL) return NULL; - if((value=ncuriquerylookup(info->uri,key)) == NULL) + if((value=ncuriquerylookup(info->dmruri,key)) == NULL) return NULL; return value; } @@ -596,3 +534,177 @@ NCD4_get_substrate(NC* nc) } else subnc = nc; return subnc; } + +/**************************************************/ +/* Allocate/Free for various structures */ + +int +NCD4_newInfo(NCD4INFO** d4infop) +{ + int ret = NC_NOERR; + NCD4INFO* info = NULL; + if((info = calloc(1,sizeof(NCD4INFO)))==NULL) + {ret = NC_ENOMEM; goto done;} + info->platform.hostlittleendian = NCD4_isLittleEndian(); + if(d4infop) {*d4infop = info; info = NULL;} +done: + if(info) NCD4_reclaimInfo(info); + return THROW(ret); +} + +/* Reclaim an NCD4INFO instance */ +void +NCD4_reclaimInfo(NCD4INFO* d4info) +{ + size_t i; + if(d4info == NULL) return; + d4info->controller = NULL; /* break link */ + nullfree(d4info->rawdmrurltext); + nullfree(d4info->dmrurltext); + ncurifree(d4info->dmruri); + freeCurl(d4info->curl); + nullfree(d4info->fileproto.filename); + NCD4_resetInfoForRead(d4info); + nullfree(d4info->substrate.filename); /* always reclaim */ + NC_authfree(d4info->auth); + nclistfree(d4info->blobs); + /* Reclaim dmr node tree */ + NCD4_reclaimMeta(d4info->dmrmetadata); + for(i=0;iresponses);i++) { + NCD4response* resp = nclistget(d4info->responses,i); + NCD4_reclaimResponse(resp); + } + nclistfree(d4info->responses); + /* Reclaim all responses */ + NCD4_resetMeta(d4info->dmrmetadata); + free(d4info); +} + +/* Reset NCD4INFO instance for new read request */ +void +NCD4_resetInfoForRead(NCD4INFO* d4info) +{ + if(d4info == NULL) return; + if(d4info->substrate.realfile + && !FLAGSET(d4info->controls.debugflags,NCF_DEBUG_COPY)) { + /* We used real file, so we need to delete the temp file + unless we are debugging. + Assume caller has done nc_close|nc_abort on the ncid. + Note that in theory, this should not be necessary since + AFAIK the substrate file is still in def mode, and + when aborted, it should be deleted. But that is not working + for some reason, so we delete it ourselves. + */ + if(d4info->substrate.filename != NULL) { + unlink(d4info->substrate.filename); + } + } + NCD4_resetMeta(d4info->dmrmetadata); + nullfree(d4info->dmrmetadata); + d4info->dmrmetadata = NULL; +} + +static void +freeCurl(NCD4curl* curl) +{ + if(curl == NULL) return; + NCD4_curlclose(curl->curl); + ncbytesfree(curl->packet); + nullfree(curl->errdata.code); + nullfree(curl->errdata.message); + free(curl); +} + +int +NCD4_newResponse(NCD4INFO* info, NCD4response** respp) +{ + int ret = NC_NOERR; + NCD4response* resp = NULL; + NC_UNUSED(info); + if((resp = calloc(1,sizeof(NCD4response)))==NULL) + {ret = NC_ENOMEM; goto done;} + resp->controller = info; + if(respp) {*respp = resp; resp = NULL;} +done: + if(resp) NCD4_reclaimResponse(resp); + return THROW(ret); +} + + +/* Reclaim an NCD4response instance */ +void +NCD4_reclaimResponse(NCD4response* d4resp) +{ + struct NCD4serial* serial = NULL; + if(d4resp == NULL) return; + serial = &d4resp->serial; + d4resp->controller = NULL; /* break link */ + + nullfree(d4resp->raw.memory); + + nullfree(serial->errdata); + nullfree(serial->dmr); + nullfree(serial->dap); + /* clear all fields */ + memset(serial,0,sizeof(struct NCD4serial)); + + nullfree(d4resp->error.parseerror); + nullfree(d4resp->error.message); + nullfree(d4resp->error.context); + nullfree(d4resp->error.otherinfo); + memset(&d4resp->error,0,sizeof(d4resp->error)); + + free(d4resp); +} + + +/* Create an empty NCD4meta object for + use in subsequent calls + (is the the right src file to hold this?) +*/ + +int +NCD4_newMeta(NCD4INFO* info, NCD4meta** metap) +{ + int ret = NC_NOERR; + NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); + if(meta == NULL) return NC_ENOMEM; + meta->allnodes = nclistnew(); +#ifdef D4DEBUG + meta->debuglevel = 1; +#endif + meta->controller = info; + meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ + if(metap) {*metap = meta; meta = NULL;} + return THROW(ret); +} + +void +NCD4_reclaimMeta(NCD4meta* dataset) +{ + int i; + if(dataset == NULL) return; + NCD4_resetMeta(dataset); + + for(i=0;iallnodes);i++) { + NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); + reclaimNode(node); + } + nclistfree(dataset->allnodes); + nclistfree(dataset->groupbyid); + nclistfree(dataset->atomictypes); + free(dataset); +} + +void +NCD4_resetMeta(NCD4meta* dataset) +{ + if(dataset == NULL) return; +#if 0 + for(i=0;iblobs);i++) { + void* p = nclistget(dataset->blobs,i); + nullfree(p); + } + nclistfree(dataset->blobs); +#endif +} diff --git a/libdap4/d4fix.c b/libdap4/d4fix.c index 82049dd90f..b76a52d28b 100644 --- a/libdap4/d4fix.c +++ b/libdap4/d4fix.c @@ -190,7 +190,7 @@ walk(NCD4node* node, NClist* sorted) */ int -NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset) +NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset, int inferredchecksumming) { int ret = NC_NOERR; NCD4mark mark = 0; @@ -214,7 +214,7 @@ NCD4_delimit(NCD4meta* compiler, NCD4node* topvar, NCD4offset* offset) topvar->data.dap4data.memory = mark; topvar->data.dap4data.size = OFFSETSIZE(offset,mark); /* extract the dap4 data checksum, if present */ - if(compiler->controller->data.inferredchecksumming) { + if(inferredchecksumming) { union ATOMICS csum; TRANSFER(csum.u8,offset,CHECKSUMSIZE); topvar->data.remotechecksum = csum.u32[0]; diff --git a/libdap4/d4http.c b/libdap4/d4http.c index 8e370d8793..13cb82a193 100644 --- a/libdap4/d4http.c +++ b/libdap4/d4http.c @@ -6,7 +6,6 @@ #include "d4includes.h" #include "d4curlfunctions.h" -static size_t WriteFileCallback(void*, size_t, size_t, void*); static size_t WriteMemoryCallback(void*, size_t, size_t, void*); static int curlerrtoncerr(CURLcode cstat); @@ -33,59 +32,6 @@ NCD4_fetchhttpcode(CURL* curl) return httpcode; } -int -NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, - d4size_t* sizep, long* filetime) -{ - int ret = NC_NOERR; - CURLcode cstat = CURLE_OK; - struct Fetchdata fetchdata; - - /* Set the URL */ - cstat = curl_easy_setopt(curl, CURLOPT_URL, (void*)url); - if (cstat != CURLE_OK) goto fail; - - /* send all data to this function */ - cstat = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteFileCallback); - if (cstat != CURLE_OK) goto fail; - - /* we pass our file to the callback function */ - cstat = curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&fetchdata); - if(cstat != CURLE_OK) goto fail; - - /* One last thing; always try to get the last modified time */ - cstat = curl_easy_setopt(curl, CURLOPT_FILETIME, (long)1); - if (cstat != CURLE_OK) goto fail; - - fetchdata.stream = stream; - fetchdata.size = 0; - cstat = curl_easy_perform(curl); - if (cstat != CURLE_OK) - {ret = NC_EDAPSVC; goto fail;} - - if (ret == NC_NOERR) { - /* return the file size*/ -#ifdef D4DEBUG - nclog(NCLOGNOTE,"filesize: %lu bytes",fetchdata.size); -#endif - if (sizep != NULL) - *sizep = fetchdata.size; - /* Get the last modified time */ - if(filetime != NULL) - cstat = curl_easy_getinfo(curl,CURLINFO_FILETIME,filetime); - if(cstat != CURLE_OK) - {ret = NC_ECURL; goto fail;} - } - return THROW(ret); - -fail: - if(cstat != CURLE_OK) { - nclog(NCLOGERR, "curl error: %s", curl_easy_strerror(cstat)); - ret = curlerrtoncerr(cstat); - } - return THROW(ret); -} - int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcodep) { @@ -155,27 +101,6 @@ NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* ht return THROW(ret); } -static size_t -WriteFileCallback(void* ptr, size_t size, size_t nmemb, void* data) -{ - size_t realsize = size * nmemb; - size_t count; - struct Fetchdata* fetchdata; - fetchdata = (struct Fetchdata*) data; - if(realsize == 0) - nclog(NCLOGWARN,"WriteFileCallback: zero sized chunk"); - count = fwrite(ptr, size, nmemb, fetchdata->stream); - if (count > 0) { - fetchdata->size += (count * size); - } else { - nclog(NCLOGWARN,"WriteFileCallback: zero sized write"); - } -#ifdef PROGRESS - nclog(NCLOGNOTE,"callback: %lu bytes",(d4size_t)realsize); -#endif - return count; -} - static size_t WriteMemoryCallback(void *ptr, size_t size, size_t nmemb, void *data) { diff --git a/libdap4/d4meta.c b/libdap4/d4meta.c index 4293b3727c..1cea62f613 100644 --- a/libdap4/d4meta.c +++ b/libdap4/d4meta.c @@ -89,75 +89,6 @@ NCD4_metabuild(NCD4meta* metadata, int ncid) return THROW(ret); } - -/* Create an empty NCD4meta object for - use in subsequent calls - (is the the right src file to hold this?) -*/ - -NCD4meta* -NCD4_newmeta(NCD4INFO* info) -{ - NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); - if(meta == NULL) return NULL; - meta->allnodes = nclistnew(); -#ifdef D4DEBUG - meta->debuglevel = 1; -#endif - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - return meta; -} - -/* Attach raw data to metadata */ -void -NCD4_attachraw(NCD4meta* meta, size_t rawsize, void* rawdata) -{ - assert(meta != NULL); - NCD4_resetSerial(&meta->serial,rawsize,rawdata); -} - -void -NCD4_setdebuglevel(NCD4meta* meta, int debuglevel) -{ - meta->debuglevel = debuglevel; -} - -void -NCD4_reclaimMeta(NCD4meta* dataset) -{ - int i; - if(dataset == NULL) return; - NCD4_resetMeta(dataset); - - for(i=0;iallnodes);i++) { - NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); - reclaimNode(node); - } - nclistfree(dataset->allnodes); - nclistfree(dataset->groupbyid); - nclistfree(dataset->atomictypes); - free(dataset); -} - -void -NCD4_resetMeta(NCD4meta* dataset) -{ - if(dataset == NULL) return; - nullfree(dataset->error.parseerror); dataset->error.parseerror = NULL; - nullfree(dataset->error.message); dataset->error.message = NULL; - nullfree(dataset->error.context); dataset->error.context = NULL; - nullfree(dataset->error.otherinfo); dataset->error.otherinfo = NULL; - NCD4_resetSerial(&dataset->serial,0,NULL); -#if 0 - for(i=0;iblobs);i++) { - void* p = nclistget(dataset->blobs,i); - nullfree(p); - } - nclistfree(dataset->blobs); -#endif -} - void reclaimNode(NCD4node* node) { @@ -676,6 +607,39 @@ savevarbyid(NCD4node* group, NCD4node* var) nclistinsert(group->group.varbyid,var->meta.id,var); } +/* Collect FQN path from var node up to and including + the root group and create an name from it +*/ +char* +NCD4_getVarFQN(NCD4node* var, const char* tail) +{ + int i; + NCD4node* x = NULL; + NClist* path = NULL; + NCbytes* fqn = NULL; + char* result; + + path = nclistnew(); + for(x=var->container;ISGROUP(x->sort);x=x->container) { + nclistinsert(path,0,x); + } + fqn = ncbytesnew(); + for(i=0;iname); + if(escaped == NULL) return NULL; + if(i > 0) ncbytesappend(fqn,'/'); + ncbytescat(fqn,escaped); + free(escaped); + } + nclistfree(path); + if(tail != NULL) + ncbytescat(fqn,tail); + result = ncbytesextract(fqn); + ncbytesfree(fqn); + return result; +} + /* Collect FQN path from node up to (but not including) the first enclosing group and create an name from it */ @@ -1180,7 +1144,7 @@ markdapsize(NCD4meta* meta) } int -NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) +NCD4_findvar(NC* ncp, int gid, int varid, NCD4node** varp, NCD4node** grpp) { int ret = NC_NOERR; NCD4INFO* info = NULL; @@ -1192,11 +1156,11 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) info = getdap(ncp); if(info == NULL) return THROW(NC_EBADID); - meta = info->substrate.metadata; + meta = info->dmrmetadata; if(meta == NULL) return THROW(NC_EBADID); /* Locate var node via (grpid,varid) */ - grp_id = GROUPIDPART(ncid); + grp_id = GROUPIDPART(gid); group = nclistget(meta->groupbyid,grp_id); if(group == NULL) return THROW(NC_EBADID); @@ -1205,7 +1169,7 @@ NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp) return THROW(NC_EBADID); if(varp) *varp = var; if(grpp) *grpp = group; - return ret; + return THROW(ret); } static int diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index 013e320928..776163a013 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -154,7 +154,7 @@ static int defineBytestringType(NCD4parser*); /* API */ int -NCD4_parse(NCD4meta* metadata) +NCD4_parse(NCD4meta* metadata, NCD4response* resp, int dapparse) { int ret = NC_NOERR; NCD4parser* parser = NULL; @@ -168,8 +168,10 @@ NCD4_parse(NCD4meta* metadata) /* Create and fill in the parser state */ parser = (NCD4parser*)calloc(1,sizeof(NCD4parser)); if(parser == NULL) {ret=NC_ENOMEM; goto done;} + parser->controller = metadata->controller; parser->metadata = metadata; - doc = ncxml_parse(parser->metadata->serial.dmr,strlen(parser->metadata->serial.dmr)); + parser->response = resp; + doc = ncxml_parse(parser->response->serial.dmr,strlen(parser->response->serial.dmr)); if(doc == NULL) {ret=NC_ENOMEM; goto done;} dom = ncxml_root(doc); parser->types = nclistnew(); @@ -178,6 +180,7 @@ NCD4_parse(NCD4meta* metadata) #ifdef D4DEBUG parser->debuglevel = 1; #endif + parser->dapparse = dapparse; /*Walk the DOM tree to build the DAP4 node tree*/ ret = traverse(parser,dom); @@ -214,9 +217,9 @@ traverse(NCD4parser* parser, ncxml_t dom) ret=parseError(parser,dom); /* Report the error */ fprintf(stderr,"DAP4 Error: http-code=%d message=\"%s\" context=\"%s\"\n", - parser->metadata->error.httpcode, - parser->metadata->error.message, - parser->metadata->error.context); + parser->response->error.httpcode, + parser->response->error.message, + parser->response->error.context); fflush(stderr); ret=NC_EDMR; goto done; @@ -847,23 +850,23 @@ parseError(NCD4parser* parser, ncxml_t errxml) char* shttpcode = ncxml_attr(errxml,"httpcode"); ncxml_t x; if(shttpcode == NULL) shttpcode = strdup("400"); - if(sscanf(shttpcode,"%d",&parser->metadata->error.httpcode) != 1) + if(sscanf(shttpcode,"%d",&parser->response->error.httpcode) != 1) nclog(NCLOGERR,"Malformed response"); nullfree(shttpcode); x=ncxml_child(errxml, "Message"); if(x != NULL) { char* txt = ncxml_text(x); - parser->metadata->error.message = (txt == NULL ? NULL : txt); + parser->response->error.message = (txt == NULL ? NULL : txt); } x = ncxml_child(errxml, "Context"); if(x != NULL) { const char* txt = ncxml_text(x); - parser->metadata->error.context = (txt == NULL ? NULL : strdup(txt)); + parser->response->error.context = (txt == NULL ? NULL : strdup(txt)); } x=ncxml_child(errxml, "OtherInformation"); if(x != NULL) { const char* txt = ncxml_text(x); - parser->metadata->error.otherinfo = (txt == NULL ? NULL : strdup(txt)); + parser->response->error.otherinfo = (txt == NULL ? NULL : strdup(txt)); } return THROW(NC_NOERR); } @@ -1321,7 +1324,7 @@ makeNode(NCD4parser* parser, NCD4node* parent, ncxml_t xml, NCD4sort sort, nc_ty record(parser,node); if(nodep) *nodep = node; done: - return ret; + return THROW(ret); } static int @@ -1652,12 +1655,19 @@ parseForwards(NCD4parser* parser, NCD4node* root) const char* mapname = (const char*)nclistget(var->mapnames,j); /* Find the corresponding variable */ NCD4node* mapref = lookupFQN(parser,mapname,NCD4_VAR); - if(mapref == NULL) + if(mapref != NULL) + PUSH(var->maps,mapref); + else if(!parser->dapparse) FAIL(NC_ENOTVAR," name does not refer to a variable: %s",mapname); - PUSH(var->maps,mapref); } } done: return THROW(ret); } + +void +NCD4_setdebuglevel(NCD4parser* parser, int debuglevel) +{ + parser->debuglevel = debuglevel; +} diff --git a/libdap4/d4read.c b/libdap4/d4read.c index 3a0e95a1d1..3a24b08ff0 100644 --- a/libdap4/d4read.c +++ b/libdap4/d4read.c @@ -19,9 +19,8 @@ See \ref copyright file for more info. /* Do conversion if this code was compiled via Vis. Studio or Mingw */ /*Forward*/ -static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, long*); +static int readpacket(NCD4INFO* state, NCURI*, NCbytes*, NCD4mode, NCD4format, int*, long*); static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet); -static int readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep); static int readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet); #ifdef HAVE_GETTIMEOFDAY @@ -38,82 +37,27 @@ deltatime(struct timeval time0,struct timeval time1) #endif int -NCD4_readDMR(NCD4INFO* state, int flags) +NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp) { int stat = NC_NOERR; - long lastmod = -1; - - if((flags & NCF_ONDISK) == 0) { - ncbytesclear(state->curl->packet); - stat = readpacket(state,state->uri,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&lastmod); - if(stat == NC_NOERR) - state->data.dmrlastmodified = lastmod; - } else { /*((flags & NCF_ONDISK) != 0) */ - NCURI* url = state->uri; - int fileprotocol = (strcmp(url->protocol,"file")==0); - if(fileprotocol) { - stat = readfiletofile(state, url, NCD4_DMR, NCD4_FORMAT_XML, state->data.ondiskfile, &state->data.datasize); - } else { - char* readurl = NULL; - int flags = 0; - if(!fileprotocol) flags |= NCURIQUERY; - flags |= NCURIENCODE; - flags |= NCURIPWD; -#ifdef FIX - ncurisetconstraints(url,state->constraint); -#endif - readurl = ncuribuild(url,NULL,".dmr.xml",NCURISVC); - if(readurl == NULL) - return THROW(NC_ENOMEM); - stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, - &state->data.datasize, &lastmod); - nullfree(readurl); - if(stat == NC_NOERR) - state->data.dmrlastmodified = lastmod; - } - } + ncbytesclear(state->curl->packet); + stat = readpacket(state,url,state->curl->packet,NCD4_DMR,NCD4_FORMAT_XML,&resp->serial.httpcode,NULL); return THROW(stat); } int -NCD4_readDAP(NCD4INFO* state, int flags) +NCD4_readDAP(NCD4INFO* state, int flags, NCURI* url, NCD4response* resp) { int stat = NC_NOERR; - long lastmod = -1; - - if((flags & NCF_ONDISK) == 0) { - ncbytesclear(state->curl->packet); - stat = readpacket(state,state->uri,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&lastmod); - if(stat) { - NCD4_seterrormessage(state->substrate.metadata, nclistlength(state->curl->packet), nclistcontents(state->curl->packet)); - goto done; - } else - state->data.daplastmodified = lastmod; - } else { /*((flags & NCF_ONDISK) != 0) */ - NCURI* url = state->uri; - int fileprotocol = (strcmp(url->protocol,"file")==0); - if(fileprotocol) { - stat = readfiletofile(state, url, NCD4_DAP, NCD4_FORMAT_NONE, state->data.ondiskfile, &state->data.datasize); - } else { - char* readurl = NULL; - int flags = 0; - if(!fileprotocol) flags |= NCURIQUERY; - flags |= NCURIENCODE; - flags |= NCURIPWD; -#ifdef FIX - ncurisetconstraints(url,state->constraint); -#endif - readurl = ncuribuild(url,NULL,".dap",NCURISVC); - if(readurl == NULL) - return THROW(NC_ENOMEM); - stat = NCD4_fetchurl_file(state->curl, readurl, state->data.ondiskfile, - &state->data.datasize, &lastmod); - nullfree(readurl); - if(stat == NC_NOERR) - state->data.daplastmodified = lastmod; - } + + ncbytesclear(state->curl->packet); + stat = readpacket(state,url,state->curl->packet,NCD4_DAP,NCD4_FORMAT_NONE,&resp->serial.httpcode,NULL); + if(stat) { + NCD4_seterrormessage(resp, nclistlength(state->curl->packet), nclistcontents(state->curl->packet)); + } else { + resp->raw.size = ncbyteslength(state->curl->packet); + resp->raw.memory = ncbytesextract(state->curl->packet); } -done: return THROW(stat); } @@ -150,7 +94,7 @@ dxxformat(int fxx, int dxx) } static int -readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, long* lastmodified) +readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4format fxx, int* httpcodep, long* lastmodified) { int stat = NC_NOERR; int fileprotocol = 0; @@ -185,7 +129,7 @@ readpacket(NCD4INFO* state, NCURI* url, NCbytes* packet, NCD4mode dxx, NCD4forma gettimeofday(&time0,NULL); #endif } - stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,&state->substrate.metadata->error.httpcode); + stat = NCD4_fetchurl(curl,fetchurl,packet,lastmodified,httpcodep); nullfree(fetchurl); if(stat) goto fail; if(FLAGSET(state->controls.flags,NCF_SHOWFETCH)) { @@ -207,37 +151,26 @@ fprintf(stderr,"readpacket: packet.size=%lu\n", return THROW(stat); } +#if 0 static int -readfiletofile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, FILE* stream, d4size_t* sizep) +readfromfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, d4size_t* sizep) { int stat = NC_NOERR; - NCbytes* packet = ncbytesnew(); size_t len; - stat = readfile(state, uri, dxx, fxx, packet); + ncbytesclear(state->curl->packet); + stat = readfile(state, uri, dxx, fxx, state->curl->packet); #ifdef D4DEBUG fprintf(stderr,"readfiletofile: packet.size=%lu\n", - (unsigned long)ncbyteslength(packet)); + (unsigned long)ncbyteslength(state->curl->packet)); #endif if(stat != NC_NOERR) goto unwind; - len = nclistlength(packet); - if(stat == NC_NOERR) { - size_t written; - fseek(stream,0,SEEK_SET); - written = fwrite(ncbytescontents(packet),1,len,stream); - if(written != len) { -#ifdef D4DEBUG -fprintf(stderr,"readfiletofile: written!=length: %lu :: %lu\n", - (unsigned long)written,(unsigned long)len); -#endif - stat = NC_EIO; - } - } + len = nclistlength(state->curl->packet); if(sizep != NULL) *sizep = len; unwind: - ncbytesfree(packet); return THROW(stat); } +#endif static int readfile(NCD4INFO* state, const NCURI* uri, NCD4mode dxx, NCD4format fxx, NCbytes* packet) @@ -365,12 +298,12 @@ readfileDAPDMR(NCD4INFO* state, const NCURI* uri, NCbytes* packet) /* Extract packet as error message; assume httpcode set */ int -NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg) +NCD4_seterrormessage(NCD4response* resp, size_t len, char* msg) { - metadata->error.message = (char*)d4alloc(len+1); - if(metadata->error.message == NULL) + resp->error.message = (char*)d4alloc(len+1); + if(resp->error.message == NULL) return THROW(NC_ENOMEM); - memcpy(metadata->error.message,msg,len); - metadata->error.message[len] = '\0'; + memcpy(resp->error.message,msg,len); + resp->error.message[len] = '\0'; return THROW(NC_ENODATA); /* slight lie */ } diff --git a/libdap4/d4read.h b/libdap4/d4read.h index 826a2217cc..39ad8d97ce 100644 --- a/libdap4/d4read.h +++ b/libdap4/d4read.h @@ -7,6 +7,6 @@ #define D4READ_H extern int NCD4_readDMR(NCD4INFO*, int flags); -extern int NCD4_readDAP(NCD4INFO*, int flags); +extern int NCD4_readDAP(NCD4INFO*, int flags, NCURI* uri); #endif /*READ_H*/ diff --git a/libdap4/d4swap.c b/libdap4/d4swap.c index ed4c8dc1d8..8568fd328c 100644 --- a/libdap4/d4swap.c +++ b/libdap4/d4swap.c @@ -14,12 +14,12 @@ the incoming data to get the endianness correct. /* Forward */ -static int walkAtomicVar(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkOpaqueVar(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); -static int walkStructArray(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); -static int walkStruct(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkSeqArray(NCD4meta*, NCD4node*, NCD4node*, NCD4offset* offset); -static int walkSeq(NCD4meta*,NCD4node*, NCD4node*, NCD4offset* offset); +static int walkAtomicVar(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkOpaqueVar(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkStructArray(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkStruct(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkSeqArray(NCD4response*, NCD4node*, NCD4node*, NCD4offset* offset,int doswap); +static int walkSeq(NCD4response*,NCD4node*, NCD4node*, NCD4offset* offset,int doswap); /**************************************************/ @@ -28,43 +28,39 @@ Assumes that compiler->swap is true; does necessary byte swapping. */ int -NCD4_swapdata(NCD4meta* compiler, NClist* topvars) +NCD4_swapdata(NCD4response* resp, NCD4node* var, int doswap) { int ret = NC_NOERR; - int i; NCD4offset* offset = NULL; - offset = BUILDOFFSET(compiler->serial.dap,compiler->serial.dapsize); - for(i=0;iserial.dap,resp->serial.dapsize); OFFSET2BLOB(var->data.dap4data,offset); switch (var->subsort) { default: - if((ret=walkAtomicVar(compiler,var,var,offset))) goto done; + if((ret=walkAtomicVar(resp,var,var,offset,doswap))) goto done; break; case NC_OPAQUE: /* The only thing we need to do is swap the counts */ - if((ret=walkOpaqueVar(compiler,var,var,offset))) goto done; + if((ret=walkOpaqueVar(resp,var,var,offset,doswap))) goto done; break; case NC_STRUCT: - if((ret=walkStructArray(compiler,var,var,offset))) goto done; + if((ret=walkStructArray(resp,var,var,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeqArray(compiler,var,var,offset))) goto done; + if((ret=walkSeqArray(resp,var,var,offset,doswap))) goto done; break; } var->data.dap4data.size = DELTA(offset,var->data.dap4data.memory); /* skip checksum, if there is one */ - if(compiler->controller->data.inferredchecksumming) + if(resp->inferredchecksumming) INCR(offset,CHECKSUMSIZE); - } done: if(offset) free(offset); return THROW(ret); } static int -walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkAtomicVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -87,7 +83,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } else { /*(typesize > 1)*/ for(i=0;ioffset; - if(compiler->swap) { + if(doswap) { switch (typesize) { case 2: swapinline16(sp); break; case 4: swapinline32(sp); break; @@ -102,7 +98,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o COUNTERTYPE count; for(i=0;iswap) + if(doswap) swapinline64(offset); count = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -114,7 +110,7 @@ walkAtomicVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } static int -walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkOpaqueVar(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -125,7 +121,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o for(i=0;iswap) + if(doswap) swapinline64(offset); count = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -135,7 +131,7 @@ walkOpaqueVar(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* o } static int -walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkStructArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -144,7 +140,7 @@ walkStructArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset for(i=0;ibasetype; switch (fieldbase->subsort) { default: - if((ret=walkAtomicVar(compiler,topvar,field,offset))) goto done; + if((ret=walkAtomicVar(resp,topvar,field,offset,doswap))) goto done; break; case NC_OPAQUE: /* The only thing we need to do is swap the counts */ - if((ret=walkOpaqueVar(compiler,topvar,field,offset))) goto done; + if((ret=walkOpaqueVar(resp,topvar,field,offset,doswap))) goto done; break; case NC_STRUCT: - if((ret=walkStructArray(compiler,topvar,field,offset))) goto done; + if((ret=walkStructArray(resp,topvar,field,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeqArray(compiler,topvar,field,offset))) goto done; + if((ret=walkSeqArray(resp,topvar,field,offset,doswap))) goto done; break; } } @@ -182,7 +178,7 @@ walkStruct(NCD4meta* compiler, NCD4node* topvar, NCD4node* structtype, NCD4offse } static int -walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* offset) +walkSeqArray(NCD4response* resp, NCD4node* topvar, NCD4node* var, NCD4offset* offset, int doswap) { int ret = NC_NOERR; d4size_t i; @@ -195,7 +191,7 @@ walkSeqArray(NCD4meta* compiler, NCD4node* topvar, NCD4node* var, NCD4offset* of for(i=0;iswap) + if(doswap) swapinline64(offset); recordcount = GETCOUNTER(offset); SKIPCOUNTER(offset); @@ -225,17 +221,17 @@ walkSeq(NCD4meta* compiler, NCD4node* topvar, NCD4node* vlentype, NCD4offset* of for(i=0;isubsort) { default: /* atomic basetype */ - if((ret=walkAtomicVar(compiler,topvar,basetype,offset))) goto done; + if((ret=walkAtomicVar(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_OPAQUE: - if((ret=walkOpaqueVar(compiler,topvar,basetype,offset))) goto done; + if((ret=walkOpaqueVar(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_STRUCT: /* We can treat each record like a structure instance */ - if((ret=walkStruct(compiler,topvar,basetype,offset))) goto done; + if((ret=walkStruct(resp,topvar,basetype,offset,doswap))) goto done; break; case NC_SEQ: - if((ret=walkSeq(compiler,topvar,basetype,offset))) goto done; + if((ret=walkSeq(resp,topvar,basetype,offset,doswap))) goto done; break; } } diff --git a/libdap4/d4util.c b/libdap4/d4util.c index 4156519305..d1ba7f5828 100644 --- a/libdap4/d4util.c +++ b/libdap4/d4util.c @@ -107,44 +107,45 @@ NCD4_makeFQN(NCD4node* node) { char* fqn = NULL; char* escaped; - int i; - NCD4node* g = node; - NClist* path = nclistnew(); - size_t estimate; - - for(estimate=0;g != NULL;g=g->container) { - estimate += strlen(g->name); - nclistinsert(path,0,g); + NCbytes* buf = ncbytesnew(); + NClist* grps = nclistnew(); + NClist* parts = nclistnew(); + NCD4node* n; + size_t i; + + /* collect all the non-groups */ + for(n=node;n;n=n->container) { + if(ISGROUP(n->sort)) + nclistinsert(grps,0,n); /* keep the correct order of groups */ + else + nclistinsert(parts,0,n); } - estimate = (estimate*2) + 2*nclistlength(path); - estimate++; /*strlcat nul*/ - fqn = (char*)malloc(estimate+1); - if(fqn == NULL) goto done; - fqn[0] = '\0'; - /* Create the group-based fqn prefix */ - /* start at 1 to avoid dataset */ - for(i=1;isort != NCD4_GROUP) break; + + /* Build grp prefix of the fqn */ + for(i=1;iname); - if(escaped == NULL) {free(fqn); fqn = NULL; goto done;} - strlcat(fqn,"/",estimate); - strlcat(fqn,escaped,estimate); + escaped = backslashEscape(n->name); + if(escaped == NULL) goto done; + ncbytescat(buf,"/"); + ncbytescat(buf,escaped); free(escaped); } /* Add in the final name part (if not group) */ - if(i < nclistlength(path)) { - int last = nclistlength(path)-1; - NCD4node* n = (NCD4node*)nclistget(path,last); - char* name = NCD4_makeName(n,"."); - strlcat(fqn,"/",estimate); - strlcat(fqn,name,estimate); - nullfree(name); + for(i=0;iname); + if(escaped == NULL) goto done; + ncbytescat(buf,(i==0?"/":".")); + ncbytescat(buf,escaped); + free(escaped); } + fqn = ncbytesextract(buf); done: - nclistfree(path); + ncbytesfree(buf); + nclistfree(grps); + nclistfree(parts); return fqn; } @@ -446,11 +447,9 @@ NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian) } void -NCD4_reporterror(NCD4INFO* state) +NCD4_reporterror(NCD4response* resp, NCURI* uri) { - NCD4meta* meta = state->substrate.metadata; char* u = NULL; - if(meta == NULL) return; - u = ncuribuild(state->uri,NULL,NULL,NCURIALL); - fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,meta->error.httpcode,meta->error.message); + u = ncuribuild(uri,NULL,NULL,NCURIALL); + fprintf(stderr,"***FAIL: url=%s httpcode=%d errmsg->\n%s\n",u,resp->serial.httpcode,resp->error.message); } diff --git a/libdap4/d4util.h b/libdap4/d4util.h index b6cbde8f07..38cda6a179 100644 --- a/libdap4/d4util.h +++ b/libdap4/d4util.h @@ -17,9 +17,12 @@ */ typedef unsigned long long d4size_t; -/* Define a counted memory marker */ +/* Define a (size, memory) pair */ typedef struct D4blob {d4size_t size; void* memory;} D4blob; +/* Empty blob constant */ +#define NULLBLOB(blob) {blob.size = 0; blob.memory = NULL;} + #define OFFSET2BLOB(blob,offset) do{(blob).size = ((offset)->limit - (offset)->base); (blob).memory = (offset)->base; }while(0) #define BLOB2OFFSET(offset,blob) do{\ (offset)->base = (blob).memory; \ diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index 9a708a5d5f..1701718004 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -11,7 +11,8 @@ #include "d4odom.h" /* Forward */ -static int getvarx(int ncid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*); +static int getvarx(int gid, int varid, NCD4INFO**, NCD4node** varp, nc_type* xtypep, size_t*, nc_type* nc4typep, size_t*); +static int mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming); int NCD4_get_vara(int ncid, int varid, @@ -22,11 +23,11 @@ NCD4_get_vara(int ncid, int varid, int ret; /* TODO: optimize since we know stride is 1 */ ret = NCD4_get_vars(ncid,varid,start,edges,NC_stride_one,value,memtype); - return ret; + return THROW(ret); } int -NCD4_get_vars(int ncid, int varid, +NCD4_get_vars(int gid, int varid, const size_t *start, const size_t *edges, const ptrdiff_t* stride, void *memoryin, nc_type xtype) { @@ -43,14 +44,14 @@ NCD4_get_vars(int ncid, int varid, int rank; size_t dimsizes[NC_MAX_VAR_DIMS]; d4size_t dimproduct; - size_t dstcount; + size_t dstpos; NCD4offset* offset = NULL; - /* Get netcdf type info */ - if((ret=getvarx(ncid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size))) + /* Get netcdf var metadata and data */ + if((ret=getvarx(gid, varid, &info, &ncvar, &xtype, &xsize, &nc4type, &nc4size))) {goto done;} - meta = info->substrate.metadata; + meta = info->dmrmetadata; nctype = ncvar->basetype; rank = nclistlength(ncvar->dims); blobs = nclistnew(); @@ -74,18 +75,18 @@ NCD4_get_vars(int ncid, int varid, odom = d4odom_new(rank,start,edges,stride,dimsizes); else odom = d4scalarodom_new(); - dstcount = 0; /* We always write into dst starting at position 0*/ - for(;d4odom_more(odom);dstcount++) { + dstpos = 0; /* We always write into dst starting at position 0*/ + for(;d4odom_more(odom);dstpos++) { void* xpos; void* dst; - d4size_t count; + d4size_t pos; - count = d4odom_next(odom); - if(count >= dimproduct) { + pos = d4odom_next(odom); + if(pos >= dimproduct) { ret = THROW(NC_EINVALCOORDS); goto done; } - xpos = ((char*)memoryin)+(xsize * dstcount); /* ultimate destination */ + xpos = ((char*)memoryin)+(xsize * dstpos); /* ultimate destination */ /* We need to compute the offset in the dap4 data of this instance; for fixed size types, this is easy, otherwise we have to walk the variable size type @@ -95,16 +96,16 @@ NCD4_get_vars(int ncid, int varid, offset = NULL; offset = BUILDOFFSET(NULL,0); BLOB2OFFSET(offset,ncvar->data.dap4data); - /* Move offset to the count'th element of the array */ + /* Move offset to the pos'th element of the array */ if(nctype->meta.isfixedsize) { - INCR(offset,(dapsize*count)); + INCR(offset,(dapsize*pos)); } else { - /* We have to walk to the count'th location in the data */ - if((ret=NCD4_moveto(meta,ncvar,count,offset))) + /* We have to walk to the pos'th location in the data */ + if((ret=NCD4_moveto(meta,ncvar,pos,offset))) {goto done;} } dst = instance; - if((ret=NCD4_fillinstance(meta,nctype,offset,&dst,blobs))) + if((ret=NCD4_movetoinstance(meta,nctype,offset,&dst,blobs))) {goto done;} if(xtype == nc4type) { /* We can just copy out the data */ @@ -132,45 +133,29 @@ NCD4_get_vars(int ncid, int varid, } static int -getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, +getvarx(int gid, int varid, NCD4INFO** infop, NCD4node** varp, nc_type* xtypep, size_t* xsizep, nc_type* nc4typep, size_t* nc4sizep) { int ret = NC_NOERR; - NC* ncp; - NCD4INFO* info; - NCD4meta* meta; - NCD4node* group; - NCD4node* var; - NCD4node* type; + NC* ncp = NULL; + NCD4INFO* info = NULL; + NCD4meta* dmrmeta = NULL; + NCD4node* group = NULL; + NCD4node* var = NULL; + NCD4node* type = NULL; nc_type xtype, actualtype; size_t instancesize, xsize; + NCURI* ceuri = NULL; /* Constrained uri */ + NCD4meta* dapmeta = NULL; + NCD4response* dapresp = NULL; - if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) + if((ret = NC_check_id(gid, (NC**)&ncp)) != NC_NOERR) goto done; info = getdap(ncp); - meta = info->substrate.metadata; - - /* If the data has not already been read and processed, then do so. */ - if(meta->serial.dap == NULL) { - size_t len = 0; - void* content = NULL; - /* (Re)Build the meta data; sets serial.rawdata */ - NCD4_resetMeta(info->substrate.metadata); - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - - if((ret=NCD4_readDAP(info, info->controls.flags.flags))) goto done; - len = ncbyteslength(info->curl->packet); - content = ncbytesextract(info->curl->packet); - NCD4_resetSerial(&meta->serial, len, content); - /* Process the data part */ - if((ret=NCD4_dechunk(meta))) goto done; - if((ret = NCD4_processdata(info->substrate.metadata))) goto done; - } - - if((ret = NCD4_findvar(ncp,ncid,varid,&var,&group))) goto done; + dmrmeta = info->dmrmetadata; + if((ret = NCD4_findvar(ncp,gid,varid,&var,&group))) goto done; type = var->basetype; actualtype = type->meta.id; instancesize = type->meta.memsize; @@ -189,6 +174,39 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, else xsize = instancesize; + /* If we already have valid data, then just return */ + if(var->data.valid) goto validated; + + /* Ok, we need to read from the server */ + + /* Add the variable to the URI, unless the URI is already constrained or is unconstrainable */ + ceuri = ncuriclone(info->dmruri); + /* append the request for a specific variable */ + if(ncuriquerylookup(ceuri,DAP4CE) == NULL && !FLAGSET(info->controls.flags,NCF_UNCONSTRAINABLE)) { + ncurisetquerykey(ceuri,strdup("dap4.ce"),NCD4_makeFQN(var)); + } + + /* Read and process the data */ + /* Setup the meta-data for the DAP */ + if((ret=NCD4_newMeta(info,&dapmeta))) goto done; + if((ret=NCD4_newResponse(info,&dapresp))) goto done; + dapresp->mode = NCD4_DAP; + nclistpush(info->responses,dapresp); + if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done; + /* Extract DMR and dechunk the data part */ + if((ret=NCD4_dechunk(dapresp))) goto done; + /* Process the dmr part */ + if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; + /* See if we are checksumming */ + if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; + /* connect variables and corresponding dap data */ + if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; + /* Process checksums and byte-order swapping */ + if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; + /* Transfer and process the data */ + if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; + +validated: /* Return relevant info */ if(infop) *infop = info; if(xtypep) *xtypep = xtype; @@ -197,8 +215,101 @@ getvarx(int ncid, int varid, NCD4INFO** infop, NCD4node** varp, if(nc4sizep) *nc4sizep = instancesize; if(varp) *varp = var; done: - if(meta->error.message != NULL) - NCD4_reporterror(info); /* Make sure the user sees this */ + if(dapmeta) NCD4_reclaimMeta(dapmeta); + ncurifree(ceuri); + if(dapresp != NULL && dapresp->error.message != NULL) + NCD4_reporterror(dapresp,ceuri); /* Make sure the user sees this */ return THROW(ret); } +#if 0 +static NCD4node* +findbyname(const char* name, NClist* nodes) +{ + int i; + for(i=0;iname)==0) + return node; + } + return NULL; +} +#endif + +static int +matchvar(NCD4meta* dmrmeta, NCD4node* dapvar, NCD4node** dmrvarp) +{ + int i,ret = NC_NOERR; + NCD4node* x = NULL; + NClist* dappath = nclistnew(); + NClist* dmrpath = nclistnew(); /* compute path for this dmr var */ + int found = 0; + NCD4node* match = NULL; + + /* Capture the dap path starting at root and ending at the dapvar (assumed to be topvar) */ + for(x=dapvar;x != NULL;x=x->container) nclistinsert(dappath,0,x); + /* Iterate over all variable nodes to find matching one */ + for(i=0;iallnodes);i++) { + NCD4node* node = (NCD4node*)nclistget(dmrmeta->allnodes,i); + if(ISVAR(node->sort) && strcmp(node->name,dapvar->name)==0) { /* possible candidate */ + int j; + found = 0; + nclistclear(dmrpath); + for(x=node;x != NULL;x=x->container) nclistinsert(dmrpath,0,x); + if(nclistlength(dmrpath) == nclistlength(dappath)) { /* same length paths */ + /* compare paths: name and sort */ + for(found=1,j=0;jsort != pdap->sort || strcmp(pdmr->name,pdap->name) != 0) + {found = 0; break;} + } + if(found) {match = node; break;} + } + } + } + if(!found) {ret = NC_EINVAL; goto done;} + if(dmrvarp) *dmrvarp = match; + +done: + nclistfree(dappath); + nclistfree(dmrpath); + return THROW(ret); +} + +/* +Map each toplevel dap var to the corresponding +toplevel dmr var and transfer necessary info; +*/ + +static int +mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming) +{ + int i, ret = NC_NOERR; + NCD4node* dmrroot = dmrmeta->root; + NClist* dmrtop = NULL; /* top variables in dmr tree */ + NCD4node* daproot = dapmeta->root; + NClist* daptop = NULL; /* top variables in dap tree */ + + /* Get top level variables from the dmr node tree */ + dmrtop = nclistnew(); + NCD4_getToplevelVars(dmrmeta,dmrroot,dmrtop); + + /* Get top level variables from the dap node tree */ + daptop = nclistnew(); + NCD4_getToplevelVars(dapmeta,daproot,daptop); + + /* Match up the dap top variables with the dmr top variables */ + for(i=0;idata = dapvar->data; + memset(&dapvar->data,0,sizeof(NCD4vardata)); + dmrvar->data.valid = 1; + } + +done: + return THROW(ret); +} diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h index 058e40ff7c..286406d771 100644 --- a/libdap4/ncd4.h +++ b/libdap4/ncd4.h @@ -77,7 +77,6 @@ EXTERNL int dsp_open(const char* path, ND4dsp** dspp); /* From d4http.c */ EXTERNL long NCD4_fetchhttpcode(CURL* curl); -EXTERNL int NCD4_fetchurl_file(CURL* curl, const char* url, FILE* stream, d4size_t* sizep, long* filetime); EXTERNL int NCD4_fetchurl(CURL* curl, const char* url, NCbytes* buf, long* filetime, int* httpcode); EXTERNL int NCD4_curlopen(CURL** curlp); EXTERNL void NCD4_curlclose(CURL* curl); @@ -85,15 +84,16 @@ EXTERNL int NCD4_fetchlastmodified(CURL* curl, char* url, long* filetime); EXTERNL int NCD4_ping(const char* url); /* From d4read.c */ -EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags); -EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags); -EXTERNL int NCD4_seterrormessage(NCD4meta* metadata, size_t len, char* msg); +EXTERNL int NCD4_readDMR(NCD4INFO* state, int flags, NCURI* url, NCD4response*); +EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags, NCURI* ceuri, NCD4response*); +EXTERNL int NCD4_seterrormessage(NCD4response*, size_t len, char* msg); /* From d4parser.c */ -EXTERNL int NCD4_parse(NCD4meta*); +EXTERNL int NCD4_parse(NCD4meta*, NCD4response*, int dapparse); EXTERNL NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); EXTERNL NCD4node* NCD4_groupFor(NCD4node* node); EXTERNL int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); +EXTERNL void NCD4_setdebuglevel(NCD4parser*,int); /* From d4printer.c */ EXTERNL int NCD4_print(NCD4meta*, NCbytes* output); @@ -104,29 +104,33 @@ EXTERNL void NCD4_attachraw(NCD4meta*, size_t size, void* rawdata); EXTERNL void NCD4_reclaimMeta(NCD4meta*); EXTERNL void NCD4_resetMeta(NCD4meta*); EXTERNL void reclaimNode(NCD4node* node); -EXTERNL void NCD4_setdebuglevel(NCD4meta*,int); EXTERNL int NCD4_metabuild(NCD4meta*, int ncid); EXTERNL size_t NCD4_computeTypeSize(NCD4meta*, NCD4node* type); EXTERNL int NCD4_findvar(NC* ncp, int ncid, int varid, NCD4node** varp, NCD4node** grpp); +EXTERNL char* NCD4_getVarFQN(NCD4node* var, const char* tail); + /* From d4chunk.c */ -EXTERNL int NCD4_dechunk(NCD4meta*); -EXTERNL int NCD4_infermode(NCD4meta* meta); +EXTERNL int NCD4_dechunk(NCD4response*); +EXTERNL int NCD4_infermode(NCD4response*); struct NCD4serial; EXTERNL void NCD4_resetSerial(struct NCD4serial* serial, size_t rawsize, void* rawdata); +EXTERNL void NCD4_moveSerial(struct NCD4serial* serial, struct NCD4serial* dst); /* From d4swap.c */ -EXTERNL int NCD4_swapdata(NCD4meta*, NClist* topvars); +EXTERNL int NCD4_swapdata(NCD4response*, NCD4node* topvar, int doswap); /* From d4fix.c */ -EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset); +EXTERNL int NCD4_delimit(NCD4meta*, NCD4node* var, NCD4offset* offset, int inferredchecksumming); EXTERNL int NCD4_moveto(NCD4meta*, NCD4node* var, d4size_t count, NCD4offset* offset); EXTERNL int NCD4_toposort(NCD4meta*); /* From d4data.c */ -EXTERNL int NCD4_processdata(NCD4meta*); -EXTERNL int NCD4_fillinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); +EXTERNL int NCD4_parcelvars(NCD4meta* meta, NCD4response* resp); +EXTERNL int NCD4_processdata(NCD4meta*,NCD4response*); +EXTERNL int NCD4_movetoinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); EXTERNL int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); +EXTERNL int NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp); /* From d4util.c */ EXTERNL d4size_t NCD4_dimproduct(NCD4node* node); @@ -141,7 +145,7 @@ EXTERNL char* NCD4_deescape(const char* esc); EXTERNL char* NCD4_entityescape(const char* s); EXTERNL size_t NCD4_elidenuls(char* s, size_t slen); EXTERNL void* NCD4_getheader(void* p, NCD4HDR* hdr, int hostlittleendian); -EXTERNL void NCD4_reporterror(NCD4INFO* state); +EXTERNL void NCD4_reporterror(NCD4response*, NCURI* uri); /* From d4dump.c */ EXTERNL void NCD4_dumpbytes(size_t size, const void* data0, int swap); @@ -163,7 +167,15 @@ EXTERNL int NCD4_convert(nc_type srctype, nc_type dsttype, char* memory0, char* /* d4file.c */ EXTERNL int NCD4_readDMRorDAP(NCD4INFO* d4info, NCD4mode mode); EXTERNL void NCD4_applyclientfragmentcontrols(NCD4INFO* d4info); -EXTERNL void NCD4_applyclientquerycontrols(NCD4INFO* d4info); +EXTERNL void NCD4_applychecksumcontrols(NCD4INFO* d4info, NCD4response*); +EXTERNL int NCD4_newInfo(NCD4INFO** d4infop); +EXTERNL void NCD4_reclaimInfo(NCD4INFO* d4info); +EXTERNL void NCD4_resetInfoforRead(NCD4INFO* d4info); +EXTERNL int NCD4_newResponse(NCD4INFO*,NCD4response** respp); +EXTERNL void NCD4_reclaimResponse(NCD4response* d4resp); +EXTERNL void NCD4_resetInfoForRead(NCD4INFO* d4info); +EXTERNL int NCD4_newMeta(NCD4INFO*,NCD4meta**); +EXTERNL void NCD4_reclaimMeta(NCD4meta*); /* ncd4dispatch.c */ struct NC_reservedatt; /*forward*/ diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 9d9cf08a1c..5f5ebbfec4 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -847,7 +847,7 @@ NCD4_inq_dim(int ncid, int dimid, char* name, size_t* lenp) if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) goto done; info = (NCD4INFO*)ncp->dispatchdata; - meta = info->substrate.metadata; + meta = info->dmrmetadata; /* Locate the dimension specified by dimid */ for(i=0;iallnodes);i++) { @@ -871,16 +871,15 @@ static int ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* value, nc_type t, const NC_reservedatt* rsvp) { int ret = NC_NOERR; - NCD4INFO* info = (NCD4INFO*)(ncp->dispatchdata); - NCD4meta* meta = info->substrate.metadata; NCD4node* var = NULL; + if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done; + if(strcmp(rsvp->name,D4CHECKSUMATTR)==0) { unsigned int* ip = (unsigned int*)value; if(varid == NC_GLOBAL) {ret = NC_EBADID; goto done;} if(t != NC_UINT) {ret = NC_EBADTYPE; goto done;} - if((ret=NCD4_findvar(ncp,ncid,varid,&var,NULL))) goto done; if(var->data.checksumattr == 0) {ret = NC_ENOTATT; goto done;} *ip = (var->data.remotechecksum); @@ -889,7 +888,7 @@ ncd4_get_att_reserved(NC* ncp, int ncid, int varid, const char* name, void* valu if(varid != NC_GLOBAL) {ret = NC_EBADID; goto done;} if(t != NC_INT) {ret = NC_EBADTYPE; goto done;} - *ip = (meta->serial.remotelittleendian?1:0); + *ip = (var->data.response->remotelittleendian?1:0); } done: return THROW(ret); @@ -925,7 +924,7 @@ static int globalinit(void) { int stat = NC_NOERR; - return stat; + return THROW(stat); } /**************************************************/ diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index 1f84a0f569..ca0a5118ac 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -50,6 +50,8 @@ typedef struct NCD4node NCD4node; typedef struct NCD4params NCD4params; typedef struct NCD4HDR NCD4HDR; typedef struct NCD4offset NCD4offset; +typedef struct NCD4vardata NCD4vardata; +typedef struct NCD4response NCD4response; /* Define the NCD4HDR flags */ /* Header flags */ @@ -228,19 +230,20 @@ struct NCD4node { int isfixedsize; /* sort == NCD4_TYPE; Is this a fixed size (recursively) type? */ d4size_t dapsize; /* size of the type as stored in the dap data; will, as a rule, be same as memsize only for types <= NC_UINT64 */ - nc_type cmpdid; /*netcdf id for the compound type created for seq type */ + nc_type cmpdid; /* netcdf id for the compound type created for seq type */ size_t memsize; /* size of a memory instance without taking dimproduct into account, but taking compound alignment into account */ d4size_t offset; /* computed structure field offset in memory */ size_t alignment; /* computed structure field alignment in memory */ } meta; - struct { /* Data compilation info */ - int flags; /* See d4data for actual flags */ + struct NCD4vardata { /* Data compilation info */ + int valid; /* 1 => this contains valid data */ D4blob dap4data; /* offset and start pos for this var's data in serialization */ - unsigned int remotechecksum; /* checksum from data as sent by server */ - unsigned int localchecksum; /* toplevel variable checksum as computed by client */ - int checksumattr; /* 1=> _DAP4_Checksum_CRC32 is defined */ - int attrchecksum; /* _DAP4_Checksum_CRC32 value */ + unsigned remotechecksum; /* toplevel per-variable checksum contained in the data */ + unsigned localchecksum; /* toplevel variable checksum as computed by client */ + int checksumattr; /* 1 => _DAP4_Checksum_CRC32 is defined */ + unsigned attrchecksum; /* _DAP4_Checksum_CRC32 value; this is the checksum computed by server */ + NCD4response* response; /* Response from which this data is taken */ } data; struct { /* Track netcdf-4 conversion info */ int isvlen; /* _edu.ucar.isvlen */ @@ -253,36 +256,13 @@ struct NCD4node { } nc4; }; -/* Tracking info about the serialized input before and after de-chunking */ -typedef struct NCD4serial { - size_t rawsize; /* |rawdata| */ - void* rawdata; - size_t dapsize; /* |dap|; this is transient */ - void* dap; /* pointer into rawdata where dap data starts */ - char* dmr;/* copy of dmr */ - char* errdata; /* null || error chunk (null terminated) */ - int httpcode; /* returned from last request */ - int hostlittleendian; /* 1 if the host is little endian */ - int remotelittleendian; /* 1 if the packet says data is little endian */ -} NCD4serial; - -/* This will be passed out of the parse */ +/* DMR information from a response; this will be passed out of the parse */ struct NCD4meta { NCD4INFO* controller; int ncid; /* root ncid of the substrate netcdf-4 file; warning: copy of NCD4Info.substrate.nc4id */ NCD4node* root; - NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */ NClist* allnodes; /*list*/ - struct Error { /* Content of any error response */ - char* parseerror; - int httpcode; - char* message; - char* context; - char* otherinfo; - } error; - int debuglevel; - NCD4serial serial; int swap; /* 1 => swap data */ /* Define some "global" (to a DMR) data */ NClist* groupbyid; /* NClist indexed by groupid >> 16; this is global */ @@ -292,9 +272,12 @@ struct NCD4meta { }; typedef struct NCD4parser { + NCD4INFO* controller; char* input; int debuglevel; + int dapparse; /* 1 => we are parsing the DAP DMR */ NCD4meta* metadata; + NCD4response* response; /* Capture useful subsets of dataset->allnodes */ NClist* types; /*list; user-defined types only*/ NClist* dims; /*list*/ @@ -303,6 +286,32 @@ typedef struct NCD4parser { NCD4node* dapopaque; /* Single non-fixed-size opaque type */ } NCD4parser; +/* Capture all the relevant info about the response to a server request */ +struct NCD4response { /* possibly processed response from a query */ + NCD4INFO* controller; /* controlling connection */ + D4blob raw; /* complete response in memory */ + int querychecksumming; /* 1 => user specified dap4.ce value */ + int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */ + int inferredchecksumming; /* 1 => either query checksum || att checksum */ + int checksumignore; /* 1 => assume checksum, but do not validate */ + int remotelittleendian; /* 1 if the packet says data is little endian */ + NCD4mode mode; /* Are we reading DMR (only) or DAP (includes DMR) */ + struct NCD4serial { + size_t dapsize; /* |dap|; this is transient */ + void* dap; /* pointer into raw where dap data starts */ + char* dmr;/* copy of dmr */ + char* errdata; /* null || error chunk (null terminated) */ + int httpcode; /* returned from last request */ + } serial; /* Dechunked and processed DAP part of the response */ + struct Error { /* Content of any error response */ + char* parseerror; + int httpcode; + char* message; + char* context; + char* otherinfo; + } error; +}; + /**************************************************/ /* Curl info */ @@ -328,28 +337,20 @@ struct NCD4curl { struct NCD4INFO { NC* controller; /* Parent instance of NCD4INFO */ - char* rawurltext; /* as given to ncd4_open */ - char* urltext; /* as modified by ncd4_open */ - NCURI* uri; /* parse of rawuritext */ + char* rawdmrurltext; /* as given to ncd4_open */ + char* dmrurltext; /* as modified by ncd4_open */ + NCURI* dmruri; /* parse of rawuritext */ NCD4curl* curl; int inmemory; /* store fetched data in memory? */ - struct { - char* memory; /* allocated memory if ONDISK is not set */ - char* ondiskfilename; /* If ONDISK is set */ - FILE* ondiskfile; /* ditto */ - d4size_t datasize; /* size on disk or in memory */ - long dmrlastmodified; - long daplastmodified; - int querychecksumming; /* 1 => user specified dap4.ce value */ - int attrchecksumming; /* 1=> _DAP4_Checksum_CRC32 is defined for at least one variable */ - int inferredchecksumming; /* 1 => either query checksum || att checksum */ - int checksumignore; /* 1 => assume checksum, but do not validate */ - } data; + NCD4meta* dmrmetadata; + NClist* responses; /* NClist all responses from this curl handle */ + struct { /* Properties that are per-platform */ + int hostlittleendian; /* 1 if the host is little endian */ + } platform; struct { int realfile; /* 1 => we created actual temp file */ char* filename; /* of the substrate file */ int nc4id; /* substrate nc4 file ncid used to hold metadata; not same as external id */ - NCD4meta* metadata; } substrate; struct { NCCONTROLS flags; @@ -362,6 +363,7 @@ struct NCD4INFO { struct { char* filename; } fileproto; + int debuglevel; NClist* blobs; }; diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 937fd8bd08..3657afedae 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -203,7 +203,7 @@ static struct NCPROTOCOLLIST { static int NC_omodeinfer(int useparallel, int omode, NCmodel*); static int check_file_type(const char *path, int omode, int use_parallel, void *parameters, NCmodel* model, NCURI* uri); static int processuri(const char* path, NCURI** urip, NClist* fraglist); -static int processmacros(NClist** fraglistp); +static int processmacros(NClist* fraglistp, NClist* expanded); static char* envvlist2string(NClist* pairs, const char*); static void set_default_mode(int* cmodep); static int parseonchar(const char* s, int ch, NClist* segments); @@ -240,8 +240,8 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv) NCURI* uri = NULL; size_t pathlen = strlen(path); char* str = NULL; - const char** ufrags; - const char** p; + const NClist* ufrags; + size_t i; if(path == NULL || pathlen == 0) {stat = NC_EURL; goto done;} @@ -263,7 +263,6 @@ processuri(const char* path, NCURI** urip, NClist* fraglenv) /* process the corresponding fragments for that protocol */ if(protolist->fragments != NULL) { - int i; tmp = nclistnew(); if((stat = parseonchar(protolist->fragments,'&',tmp))) goto done; for(i=0;isubstitute) ncurisetprotocol(uri,protolist->substitute); /* capture the fragments of the url */ - ufrags = ncurifragmentparams(uri); - if(ufrags != NULL) { - for(p=ufrags;*p;p+=2) { - const char* key = p[0]; - const char* value = p[1]; - nclistpush(fraglenv,nulldup(key)); - value = (value==NULL?"":value); - nclistpush(fraglenv,strdup(value)); - } + ufrags = (const NClist*)ncurifragmentparams(uri); + for(i=0;i 0) { + for(i=0;iname;macros++) { if(strcmp(macros->name,key)==0) { @@ -472,14 +462,8 @@ processmacros(NClist** fraglenvp) nclistpush(expanded,strdup(key)); nclistpush(expanded,strdup(value)); } - nullfree(key); - nullfree(value); } - *fraglenvp = expanded; expanded = NULL; -done: - nclistfreeall(expanded); - nclistfreeall(fraglenv); return check(stat); } @@ -495,8 +479,6 @@ processinferences(NClist* fraglenv) int i; char* newmodeval = NULL; - if(fraglenv == NULL || nclistlength(fraglenv) == 0) goto done; - /* Get "mode" entry */ if((modeval = getmodekey(fraglenv))==NULL) goto done; @@ -534,7 +516,7 @@ printlist(newmodes,"processinferences: new mode list"); nextmodes = tmp; tmp = NULL; } - /* cleanup any unused elements in currenmodes */ + /* cleanup any unused elements in currentmodes */ nclistclearall(currentmodes); /* Ensure no duplicates */ @@ -689,21 +671,15 @@ collectallkeys(NClist* fraglenv, NClist* allkeys) /* Given a fragment envv list, coalesce duplicate keys and remove duplicate values*/ static int -cleanfragments(NClist** fraglenvp) +cleanfragments(NClist* fraglenv, NClist* newlist) { int i,stat = NC_NOERR; - NClist* fraglenv = NULL; NClist* tmp = NULL; NClist* allkeys = NULL; - NClist* newlist = NULL; NCbytes* buf = NULL; char* key = NULL; char* value = NULL; - if(fraglenvp == NULL || nclistlength(*fraglenvp) == 0) return NC_NOERR; - fraglenv = *fraglenvp; /* take control of this list */ - *fraglenvp = NULL; - newlist = nclistnew(); buf = ncbytesnew(); allkeys = nclistnew(); tmp = nclistnew(); @@ -723,13 +699,10 @@ cleanfragments(NClist** fraglenvp) nclistpush(newlist,value); nclistclear(tmp); } - *fraglenvp = newlist; newlist = NULL; done: nclistfree(allkeys); nclistfree(tmp); ncbytesfree(buf); - nclistfreeall(fraglenv); - nclistfreeall(newlist); return check(stat); } @@ -869,7 +842,8 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void char* sfrag = NULL; const char* modeval = NULL; char* abspath = NULL; - + NClist* tmp = NULL; + /* Phase 1: 1. convert special protocols to http|https 2. begin collecting fragments @@ -882,13 +856,20 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void #endif /* Phase 2: Expand macros and add to fraglenv */ - if((stat = processmacros(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = processmacros(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; #ifdef DEBUG printlist(fraglenv,"processmacros"); #endif - /* Cleanup the fragment list */ - if((stat = cleanfragments(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = cleanfragments(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; /* Phase 2a: Expand mode inferences and add to fraglenv */ if((stat = processinferences(fraglenv))) goto done; @@ -897,7 +878,11 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void #endif /* Phase 3: coalesce duplicate fragment keys and remove duplicate values */ - if((stat = cleanfragments(&fraglenv))) goto done; + nclistfreeall(tmp); + tmp = nclistnew(); + if((stat = cleanfragments(fraglenv,tmp))) goto done; + nclistfreeall(fraglenv); + fraglenv = tmp; tmp = NULL; #ifdef DEBUG printlist(fraglenv,"cleanfragments"); #endif @@ -947,13 +932,12 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void /* Phase 6: Process the non-mode keys to see if we can tell the formatx */ if(!modelcomplete(model)) { - const char** p = ncurifragmentparams(uri); /* envv format */ - if(p != NULL) { - for(;*p;p+=2) { - const char* key = p[0]; - const char* value = p[1];; - if((stat=processfragmentkeys(key,value,model))) goto done; - } + size_t i; + NClist* p = (NClist*)ncurifragmentparams(uri); /* envv format */ + for(i=0;i 0) { - nclistpush(params,NULL); - tmp.fraglist = nclistextract(params); - } else - tmp.fraglist = NULL; + tmp.fraglist = params; + params = NULL; + /* Parse the query */ if(tmp.query != NULL) { if(parselist(tmp.query,querylist) != NC_NOERR) {THROW(NC_EURL);} - if(nclistlength(querylist) > 0) { - nclistpush(querylist,NULL); - tmp.querylist = nclistextract(querylist); - } + tmp.querylist = querylist; + querylist = NULL; } /* Now parse the core of the url */ @@ -312,7 +302,7 @@ ncuriparse(const char* uri0, NCURI** durip) pathchar = EOFCHAR; } else { /* assume there should be a host section */ /* We already extracted the query and/or fragment sections above, - splocate the end of the host section and therefore the start + so locate the end of the host section and therefore the start of the path. */ tmp.host = p; @@ -430,9 +420,9 @@ ncuriparse(const char* uri0, NCURI** durip) freestringlist(params); freestringlist(querylist); if(tmp.fraglist) - freestringvec(tmp.fraglist); + nclistfreeall(tmp.fraglist); if(tmp.querylist) - freestringvec(tmp.querylist); + nclistfreeall(tmp.querylist); return ret; } @@ -450,6 +440,7 @@ freestringlist(NClist* list) } } +#if 0 static void freestringvec(char** list) { @@ -459,6 +450,7 @@ freestringvec(char** list) nullfree(list); } } +#endif void ncurifree(NCURI* duri) @@ -473,8 +465,8 @@ ncurifree(NCURI* duri) nullfree(duri->path); nullfree(duri->query); nullfree(duri->fragment); - freestringvec(duri->querylist); - freestringvec(duri->fraglist); + nclistfreeall(duri->querylist); + nclistfreeall(duri->fraglist); free(duri); } @@ -510,21 +502,14 @@ int ncurisetquery(NCURI* duri,const char* query) { int ret = NC_NOERR; - freestringvec(duri->querylist); + nclistfreeall((NClist*)duri->querylist); nullfree(duri->query); duri->query = NULL; duri->querylist = NULL; if(query != NULL && strlen(query) > 0) { - NClist* params = nclistnew(); - duri->query = strdup(query); - ret = parselist(duri->query,params); - if(ret != NC_NOERR) - {THROW(NC_EURL);} - nclistpush(params,NULL); - duri->querylist = nclistextract(params); - nclistfree(params); + duri->query = strdup(query); + ensurequerylist(duri); } -done: return ret; } @@ -533,12 +518,13 @@ int ncurisetfragments(NCURI* duri,const char* fragments) { int ret = NC_NOERR; - freestringvec(duri->fraglist); + nclistfreeall((NClist*)duri->fraglist); nullfree(duri->fragment); duri->fragment = NULL; duri->fraglist = NULL; if(fragments != NULL && strlen(fragments) > 0) { duri->fragment = strdup(fragments); + ensurefraglist(duri); } return ret; } @@ -559,17 +545,20 @@ ncurisetfragmentkey(NCURI* duri,const char* key, const char* value) { int ret = NC_NOERR; int pos = -1; - char* newlist = NULL; ensurefraglist(duri); pos = ncfind(duri->fraglist, key); - if(pos < 0) return NC_EINVAL; /* does not exist */ - nullfree(duri->fraglist[pos+1]); - duri->fraglist[pos+1] = strdup(value); + if(pos < 0) { /* does not exist */ + if(duri->fraglist == NULL) duri->fraglist = nclistnew(); + nclistpush(duri->fraglist,key); + nclistpush(duri->fraglist,value); + } else { + nullfree(nclistget(duri->fraglist,pos+1)); + nclistset(duri->fraglist,pos+1,strdup(value)); + } /* Rebuild the fragment */ - if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done; - nullfree(duri->fragment); - duri->fragment = newlist; newlist = NULL; + nullfree(duri->fragment); duri->fragment = NULL; + if((ret = ensurefraglist(duri))) goto done; done: return ret; } @@ -579,25 +568,67 @@ int ncuriappendfragmentkey(NCURI* duri,const char* key, const char* value) { int ret = NC_NOERR; - int len; int pos = -1; - char* newlist = NULL; ensurefraglist(duri); pos = ncfind(duri->fraglist, key); if(pos < 0) { /* does not exist */ - if((ret = extendenvv(&duri->fraglist,2,&len))) goto done; - duri->fraglist[len] = strdup(key); - duri->fraglist[len+1] = nulldup(value); - duri->fraglist[len+2] = NULL; + nclistpush((NClist*)duri->fraglist,strdup(key)); + nclistpush((NClist*)duri->fraglist,nulldup(value)); } else { - nullfree(duri->fraglist[pos+1]); - duri->fraglist[pos+1] = strdup(value); + nullfree(nclistget(duri->fraglist,pos+1)); + nclistset(duri->fraglist,pos+1,nulldup(value)); } /* Rebuild the fragment */ - if((ret = unparselist((const char**)duri->fraglist,"#",0,&newlist))) goto done; - nullfree(duri->fragment); - duri->fragment = newlist; newlist = NULL; + nullfree(duri->fraglist); duri->fraglist = NULL; + if((ret=ensurefraglist(duri))) goto done; +done: + return ret; +} + +/* Replace a specific query key*/ +int +ncurisetquerykey(NCURI* duri,const char* key, const char* value) +{ + int ret = NC_NOERR; + int pos = -1; + + ensurequerylist(duri); + pos = ncfind(duri->querylist, key); + if(pos < 0) { /* does not exist */ + if(duri->querylist == NULL) duri->querylist = nclistnew(); + nclistpush(duri->querylist,key); + nclistpush(duri->querylist,value); + } else { + nullfree(nclistget(duri->querylist,pos+1)); + nclistset(duri->querylist,pos+1,strdup(value)); + } + /* Rebuild the query */ + nullfree(duri->query); duri->query = NULL; + if((ret = ensurequerylist(duri))) goto done; +done: + return ret; +} + +/* Replace or add a specific query key*/ +int +ncuriappendquerykey(NCURI* duri,const char* key, const char* value) +{ + int ret = NC_NOERR; + int pos = -1; + + ensurequerylist(duri); + pos = ncfind(duri->querylist, key); + if(pos < 0) { /* does not exist */ + nclistpush((NClist*)duri->querylist,strdup(key)); + nclistpush((NClist*)duri->querylist,nulldup(value)); + } else { + nullfree(nclistget(duri->querylist,pos+1)); + nclistset(duri->querylist,pos+1,nulldup(value)); + } + /* Rebuild the query */ + nullfree(duri->querylist); duri->querylist = NULL; + if((ret=ensurequerylist(duri))) goto done; done: return ret; } @@ -734,11 +765,10 @@ ncurifragmentlookup(NCURI* uri, const char* key) int i; char* value = NULL; if(uri == NULL || key == NULL) return NULL; - ensurefraglist(uri); + if(ensurefraglist(uri)) return NULL; i = ncfind(uri->fraglist,key); - if(i < 0) - return NULL; - value = uri->fraglist[(2*i)+1]; + if(i < 0) return NULL; + value = nclistget(uri->fraglist,i+1); return value; } @@ -747,20 +777,21 @@ ncuriquerylookup(NCURI* uri, const char* key) { int i; char* value = NULL; - if(uri == NULL || key == NULL || uri->querylist == NULL) return NULL; + if(uri == NULL || key == NULL) return NULL; + if(ensurequerylist(uri)) return NULL; i = ncfind(uri->querylist,key); - if(i < 0) - return NULL; - value = uri->querylist[(2*i)+1]; + if(i < 0) return NULL; + value = nclistget(uri->querylist,i+1); return value; } +#if 0 /* Obtain the complete list of fragment pairs in envv format */ const char** ncurifragmentparams(NCURI* uri) { ensurefraglist(uri); - return (const char**)uri->fraglist; + return (const char**)nclistcontents(uri->fraglist; } /* Obtain the complete list of query pairs in envv format */ @@ -771,7 +802,6 @@ ncuriqueryparams(NCURI* uri) return (const char**)uri->querylist; } -#if 0 int ncuriremoveparam(NCURI* uri, const char* key) { @@ -796,19 +826,18 @@ ncuriremoveparam(NCURI* uri, const char* key) case insensitive */ static int -ncfind(char** params, const char* key) +ncfind(NClist* params, const char* key) { int i; - char** p; if(key == NULL) return -1; if(params == NULL) return -1; - for(i=0,p=params;*p;p+=2,i++) { - if(strcasecmp(key,*p)==0) return i; + for(i=0;i 0) { + ncbytescat(buf,p0); + if(p1 != NULL && strlen(p1) > 0) { ncbytescat(buf,"="); if(encode) { - char* encoded = ncuriencodeonly(p[1],queryallow); + char* encoded = ncuriencodeonly(p1,queryallow); ncbytescat(buf,encoded); nullfree(encoded); } else - ncbytescat(buf,p[1]); + ncbytescat(buf,p1); } } - if(svecp) {*svecp = ncbytesextract(buf);} done: - ncbytesfree(buf); return stat; } @@ -1171,24 +1199,32 @@ static int ensurefraglist(NCURI* uri) { int stat = NC_NOERR; - int nofrag = 0; - int nolist = 0; - NClist* fraglist = NULL; + int hastext = 0; + int haslist = 0; + NClist* fraglist = nclistnew(); NCbytes* frag = NULL; - if(uri->fragment == NULL || strlen(uri->fragment) == 0) - {nullfree(uri->fragment); uri->fragment = NULL; nofrag=1;} - if(uri->fraglist == NULL) nolist = 1; - if(nolist && !nofrag) { - fraglist = nclistnew(); + if(nulllen(uri->fragment) == 0) + {nullfree(uri->fragment); uri->fragment = NULL; hastext=0;} + else hastext = 1; + if(nclistlength((NClist*)uri->fraglist) == 0) + {nclistfree((NClist*)uri->fraglist); uri->fraglist = NULL; haslist=0;} + else haslist = 1; + + /* Four cases: */ + if(!haslist && !hastext) { + /* do nothing */ + } else if(!haslist && hastext) { if((stat = parselist(uri->fragment,fraglist))) goto done; removedups(fraglist); - uri->fraglist = nclistextract(fraglist); - } else if(!nolist && nofrag) { + uri->fraglist = fraglist; fraglist = NULL; + } else if(haslist && !hastext) { /* Create the fragment string from fraglist */ frag = ncbytesnew(); - buildlist((const char**)uri->fraglist,0,frag); /* do not encode */ + if((stat=unparselist((const NClist*)uri->fraglist,NULL,0,frag))) goto done; /* do not encode */ uri->fragment = ncbytesextract(frag); + } else if(haslist && hastext) { + /* assume already consistent */ } done: @@ -1201,24 +1237,32 @@ static int ensurequerylist(NCURI* uri) { int stat = NC_NOERR; - int noquery = 0; - int nolist = 0; - NClist* querylist = NULL; + int hastext = 0; + int haslist = 0; + NClist* querylist = nclistnew(); NCbytes* query = NULL; - if(uri->query == NULL || strlen(uri->query) == 0) - {nullfree(uri->query); uri->query = NULL; noquery=1;} - if(uri->querylist == NULL) nolist = 1; - if(nolist && !noquery) { - querylist = nclistnew(); + if(nulllen(uri->query) == 0) + {nullfree(uri->query); uri->query = NULL; hastext=0;} + else hastext = 1; + if(nclistlength((NClist*)uri->querylist) == 0) + {nclistfree((NClist*)uri->querylist); uri->querylist = NULL; haslist=0;} + else haslist = 1; + + /* Four cases: */ + if(!haslist && !hastext) { + /* do nothing */ + } else if(!haslist && hastext) { if((stat = parselist(uri->query,querylist))) goto done; removedups(querylist); - uri->querylist = nclistextract(querylist); - } else if(!nolist && noquery) { + uri->querylist = querylist; querylist = NULL; + } else if(haslist && !hastext) { /* Create the query string from querylist */ query = ncbytesnew(); - buildlist((const char**)uri->querylist,0,query); /* do not encode */ + if((stat=unparselist((const NClist*)uri->querylist,NULL,0,query))) goto done; /* do not encode */ uri->query = ncbytesextract(query); + } else if(haslist && hastext) { + /* assume consistent */ } done: @@ -1242,30 +1286,9 @@ removedups(NClist* list) } } } - /* NULL terminate the list */ - nclistpush(list,NULL); -} - -static void -buildlist(const char** list, int encode, NCbytes* buf) -{ - const char** p; - int first = 1; - for(p=list;*p;p+=2,first=0) { - if(!first) ncbytescat(buf,"&"); - ncbytescat(buf,p[0]); - if(p[1] != NULL && strlen(p[1]) > 0) { - ncbytescat(buf,"="); - if(encode) { - char* encoded = ncuriencodeonly(p[1],queryallow); - ncbytescat(buf,encoded); - nullfree(encoded); - } else - ncbytescat(buf,p[1]); - } - } } +#if 0 static int extendenvv(char*** envvp, int amount, int* oldlenp) { @@ -1281,6 +1304,7 @@ extendenvv(char*** envvp, int amount, int* oldlenp) *envvp = envv; envv = NULL; return NC_NOERR; } +#endif /* Use for gdb debug */ char* @@ -1288,3 +1312,20 @@ ncuriunescape(const char* s) { return ncuridecodepartial(s,ascii); } + +/* Get the actual list of queryies */ +void* +ncuriqueryparams(NCURI* uri) +{ + ensurequerylist(uri); + return uri->querylist; +} + +/* Get the actual list of frags */ +void* +ncurifragmentparams(NCURI* uri) +{ + ensurefraglist(uri); + return uri->fraglist; +} + diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index 966adfbfbe..9e22f73d42 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -22,7 +22,7 @@ static int applycontrols(NCZ_FILE_INFO_T* zinfo); */ int -ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** controls) +ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls) { int stat = NC_NOERR; NCZ_FILE_INFO_T* zinfo = NULL; @@ -32,7 +32,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr NCjson* json = NULL; char* key = NULL; - ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_envv(controls):"null")); + ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_env(controls):"null")); nc = (NC*)file->controller; @@ -52,7 +52,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr zinfo->creating = 1; zinfo->common.file = file; zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zinfo->envv_controls=NCZ_clonestringvec(0,controls)) == NULL) + if((zinfo->controllist=nclistclone(controls,1)) == NULL) {stat = NC_ENOMEM; goto done;} /* fill in some of the zinfo and zroot fields */ @@ -94,7 +94,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr */ int -ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) +ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls) { int stat = NC_NOERR; NC* nc = NULL; @@ -126,7 +126,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) zinfo->creating = 0; zinfo->common.file = file; zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zinfo->envv_controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/ + if((zinfo->controllist=nclistclone(controls,1)) == NULL) {stat = NC_ENOMEM; goto done;} zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT; @@ -294,12 +294,13 @@ ncz_open_rootgroup(NC_FILE_INFO_T* dataset) static const char* -controllookup(const char** envv_controls, const char* key) +controllookup(NClist* controls, const char* key) { - const char** p; - for(p=envv_controls;*p;p+=2) { - if(strcasecmp(key,*p)==0) { - return p[1]; + int i; + for(i=0;ienvv_controls,"mode")) != NULL) { + if((value = controllookup(zinfo->controllist,"mode")) != NULL) { if((stat = NCZ_comma_parse(value,modelist))) goto done; } /* Process the modelist first */ @@ -337,11 +338,11 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) zinfo->controls.flags &= (~noflags); /* Process other controls */ - if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) { + if((value = controllookup(zinfo->controllist,"log")) != NULL) { zinfo->controls.flags |= FLAG_LOGGING; ncsetloglevel(NCLOGNOTE); } - if((value = controllookup((const char**)zinfo->envv_controls,"show")) != NULL) { + if((value = controllookup(zinfo->controllist,"show")) != NULL) { if(strcasecmp(value,"fetch")==0) zinfo->controls.flags |= FLAG_SHOWFETCH; } diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 8b3b37ca0e..22dd2d1cfc 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -26,8 +26,8 @@ struct ZCVT { #define zcvt_empty {0,0,0.0,NULL} /* zarr.c */ -EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls); -EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls); +EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, NClist* controls); +EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, NClist* controls); EXTERNL int ncz_del_attr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name); /* HDF5 Mimics */ diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index b70e86f794..9baa0fd633 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -49,7 +49,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort) if((stat = nczmap_close(zinfo->map,(abort && zinfo->creating)?1:0))) goto done; - NCZ_freestringvec(0,zinfo->envv_controls); + nclistfreeall(zinfo->controllist); NC_authfree(zinfo->auth); nullfree(zinfo); diff --git a/libnczarr/zcreate.c b/libnczarr/zcreate.c index 3243b6f8c7..bf7c1db3e5 100644 --- a/libnczarr/zcreate.c +++ b/libnczarr/zcreate.c @@ -30,7 +30,7 @@ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_DISKLESS|NC_64BIT * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_create_file(const char *path, int cmode, size_t initialsz, const char** controls, int ncid) +ncz_create_file(const char *path, int cmode, size_t initialsz, NClist* controls, int ncid) { int retval = NC_NOERR; NC_FILE_INFO_T* h5 = NULL; diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 50ff91e446..14c8079f0f 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -144,7 +144,7 @@ typedef struct NCZ_FILE_INFO { } zarr; int creating; /* 1=> created 0=>open */ int native_endianness; /* NC_ENDIAN_LITTLE | NC_ENDIAN_BIG */ - char** envv_controls; /* Envv format */ + NClist* controllist; /* Envv format */ struct Controls { size64_t flags; # define FLAG_PUREZARR 1 diff --git a/libnczarr/zopen.c b/libnczarr/zopen.c index 8d8644e0fa..2171f7b23d 100644 --- a/libnczarr/zopen.c +++ b/libnczarr/zopen.c @@ -64,7 +64,7 @@ check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic) * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_open_file(const char *path, int mode, const char** controls, int ncid) +ncz_open_file(const char *path, int mode, NClist* controls, int ncid) { int stat = NC_NOERR; NC_FILE_INFO_T *h5 = NULL; From fe3ee028596c32debf22360e63126af4fa542365 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sun, 8 Oct 2023 20:40:24 -0600 Subject: [PATCH 18/77] Update release notes --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ed19b87f71..3a82aa9a14 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD -* Improve the speed and data quantity for DAP4 queries. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Improve the speed and data quantity for DAP4 queries. See [Github #2765](https://github.com/Unidata/netcdf-c/pull/2765). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). From fabd3b2b99de51c4534d36aa2cefc987da4f79e4 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 9 Oct 2023 10:47:39 +0100 Subject: [PATCH 19/77] Fix bug with displaying log messages One of the changes in df3636b introduced the ability to filter log messages by level. This fixes a typo that flipped the intended filtering, so that `NCLOGOFF` _enabled_ all messages instead of _disabling_ them. --- libdispatch/nclog.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index 26472fdfe0..6f477c18d8 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -82,9 +82,11 @@ ncloginit(void) } /*! -Enable/Disable logging. +Enable logging messages to a given level. Set to NCLOGOFF to disable +all messages, NCLOGERR for errors only, NCLOGWARN for warnings and +errors, and so on -\param[in] tf If 1, then turn on logging, if 0, then turn off logging. +\param[in] level Messages above this level are ignored \return The previous value of the logging flag. */ @@ -136,8 +138,9 @@ ncvlog(int level, const char* fmt, va_list ap) const char* prefix; if(!nclogginginitialized) ncloginit(); - if(nclog_global.loglevel > level || nclog_global.nclogstream == NULL) - return; + if(nclog_global.loglevel < level || nclog_global.nclogstream == NULL) { + return; + } prefix = nctagname(level); fprintf(nclog_global.nclogstream,"%s: ",prefix); if(fmt != NULL) { From 5f5f908a9616eefbde5d876176b5efe61b74119b Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Mon, 9 Oct 2023 10:18:04 -0600 Subject: [PATCH 20/77] Remove vestigial file glob stanza --- dap4_test/CMakeLists.txt | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/dap4_test/CMakeLists.txt b/dap4_test/CMakeLists.txt index a44c5eb1dc..ab16df6290 100644 --- a/dap4_test/CMakeLists.txt +++ b/dap4_test/CMakeLists.txt @@ -78,15 +78,4 @@ ENDIF(ENABLE_TESTS) #FILE(COPY ./cdltestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}) #FILE(COPY ./rawtestfiles DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}) -## Specify files to be distributed by 'make dist' -FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*.c ${CMAKE_CURRENT_SOURCE_DIR}/*.h ${CMAKE_CURRENT_SOURCE_DIR}/*.sh -${CMAKE_CURRENT_SOURCE_DIR}/cdltestfiles -${CMAKE_CURRENT_SOURCE_DIR}/rawtestfiles -${CMAKE_CURRENT_SOURCE_DIR}/baseline -${CMAKE_CURRENT_SOURCE_DIR}/baselineraw -${CMAKE_CURRENT_SOURCE_DIR}/baselineremote -#${CMAKE_CURRENT_SOURCE_DIR}/baselinerthredds -${CMAKE_CURRENT_SOURCE_DIR}/baselinehyrax -) -SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt Makefile.am) -ADD_EXTRA_DIST("${CUR_EXTRA_DIST}") + From 24e49b818e60f43083147cdf50551a816882c0d7 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 12 Oct 2023 16:22:47 -0600 Subject: [PATCH 21/77] Update internal tinyxml2 code to the latest version Periodic update of the internal tinyxml2 code for processing XML. This also required some changes to the "tinyxml2" target in libncxml/Makefile.am to modify the code to compile in the netcdf-c environment. --- include/ncconfigure.h | 2 +- libncxml/Makefile.am | 19 ++++++++++++++++--- libncxml/tinyxml2.cpp | 6 ++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/include/ncconfigure.h b/include/ncconfigure.h index cad59c9e21..496ec59e03 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -61,7 +61,7 @@ extern "C" { #endif /* WARNING: in some systems, these functions may be defined as macros, so check */ -#ifndef HAVE_STRDUP +#if ! defined(HAVE_STRDUP) || defined(__CYGWIN__) #ifndef strdup char* strdup(const char*); #endif diff --git a/libncxml/Makefile.am b/libncxml/Makefile.am index 3866eae5ba..f5fab3b101 100644 --- a/libncxml/Makefile.am +++ b/libncxml/Makefile.am @@ -35,7 +35,20 @@ tinyxml2:: rm -fr ./tinyxml2 ./license.txt git clone --depth=1 ${REPO} cat tinyxml2/LICENSE.txt > ./license.txt - cat tinyxml2/tinyxml2.h > ./tinyxml2.h - sed -e 's/__BORLANDC__/__APPLE__/' < tinyxml2/tinyxml2.cpp \ - | sed -e 's/ptrdiff_t/long/g' > ./tinyxml2.cpp + tr -d '\r' < tinyxml2/tinyxml2.h > tinyxml2.h + cat tinyxml2/tinyxml2.cpp \ + | sed -e 's/__BORLANDC__/__APPLE__/' \ + | sed -e 's/ptrdiff_t/long/g' \ + | sed -e '/^static[ ]*FILE[*][ ]*callfopen(/i\ +\#if 0' \ + | sed -e '/^void[ ]*XMLDocument::DeleteNode(/i\ +\#endif /*0*/\ +' \ + | sed -e '/^XMLError[ ]*XMLDocument::LoadFile([ ]*const[ ]*char[*]/i\ +\#if 0' \ + | sed -e '/^XMLError[ ]*XMLDocument::Parse(/i\ +\#endif /*0*/\ +' \ + | tr -d '\r' \ + | cat > ./tinyxml2.cpp rm -fr tinyxml2 diff --git a/libncxml/tinyxml2.cpp b/libncxml/tinyxml2.cpp index 4f761ad319..c0d16eda0f 100644 --- a/libncxml/tinyxml2.cpp +++ b/libncxml/tinyxml2.cpp @@ -2283,6 +2283,7 @@ XMLUnknown* XMLDocument::NewUnknown( const char* str ) return unk; } +#if 0 static FILE* callfopen( const char* filepath, const char* mode ) { TIXMLASSERT( filepath ); @@ -2299,6 +2300,8 @@ static FILE* callfopen( const char* filepath, const char* mode ) return fp; } +#endif /*0*/ + void XMLDocument::DeleteNode( XMLNode* node ) { TIXMLASSERT( node ); TIXMLASSERT(node->_document == this ); @@ -2317,6 +2320,7 @@ void XMLDocument::DeleteNode( XMLNode* node ) { } +#if 0 XMLError XMLDocument::LoadFile( const char* filename ) { if ( !filename ) { @@ -2420,6 +2424,8 @@ XMLError XMLDocument::SaveFile( FILE* fp, bool compact ) } +#endif /*0*/ + XMLError XMLDocument::Parse( const char* xml, size_t nBytes ) { Clear(); From 0701250bbcc4aefc7a7a1e5a3a4faae3ba698e0d Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 12 Oct 2023 16:25:18 -0600 Subject: [PATCH 22/77] Update release notes --- RELEASE_NOTES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index c92394242d..245bc2834c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Update the internal copy of tinyxml2 to latest code. See [Github #2771](https://github.com/Unidata/netcdf-c/pull/2771). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737). From 84af03eaa2345a8e4a51b65b6babef20906da89e Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 13 Oct 2023 11:32:27 -0600 Subject: [PATCH 23/77] Working on the last few s3 issues currently manifesting using S3_INTERNAL and WITH_S3_TESTING on Windows and MSYS2. --- libdispatch/dpathmgr.c | 1 + nczarr_test/run_interop.sh | 4 ++++ nczarr_test/run_notzarr.sh | 5 +++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libdispatch/dpathmgr.c b/libdispatch/dpathmgr.c index 14a6bb1a6e..dca9f7f80f 100644 --- a/libdispatch/dpathmgr.c +++ b/libdispatch/dpathmgr.c @@ -168,6 +168,7 @@ NCpathcvt(const char* inpath) } nullfree(tmp1); clearPath(&inparsed); + //fprintf(stderr,">>> ncpathcvt: inpath=%s result=%s\n",inpath,result); return result; } diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index 3579c37384..5f85e6b0cd 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -e + if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh @@ -43,6 +45,7 @@ testcasezip() { } testcases3() { + set -x echo -e "\to Running S3 Testcase:\t$1\t$2" zext=s3 base=$1 @@ -53,6 +56,7 @@ testcases3() { ${NCDUMP} -v "/group_with_dims/var2D" $flags $url > tmp_${base}_${zext}.cdl # Find the proper ref file diff -b ${ISOPATH}/ref_${base}_2d.cdl tmp_${base}_${zext}.cdl + set +x } testallcases() { diff --git a/nczarr_test/run_notzarr.sh b/nczarr_test/run_notzarr.sh index 7eb8a01ac1..86bd570dab 100755 --- a/nczarr_test/run_notzarr.sh +++ b/nczarr_test/run_notzarr.sh @@ -1,5 +1,6 @@ #!/bin/sh - +#set -x +#set -e if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh @@ -29,7 +30,7 @@ cp ${srcdir}/ref_notzarr.tar.gz . gunzip ref_notzarr.tar.gz tar -xf ref_notzarr.tar if test "x$FEATURE_S3TESTS" = xyes ; then - ${execdir}/s3util -f notzarr.file/notzarr.txt -u "https://${URL}" -k "/${S3ISOPATH}/notzarr.s3/notzarr.txt" upload + ${execdir}/s3util -f notzarr.file/notzarr.txt -u "https://${URL}" -k "//${S3ISOPATH}/notzarr.s3/notzarr.txt" upload fi echo "Test empty file" From 4bd9c2e8b041c5fb08c1819b8cc6efaaea01f502 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 13 Oct 2023 15:02:58 -0600 Subject: [PATCH 24/77] Removed leading slash on run_interop in nczarr_test. --- nczarr_test/run_interop.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index 5f85e6b0cd..ed22197112 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -52,8 +52,9 @@ testcases3() { mode=$2 rm -f tmp_${base}_${zext}.cdl url="https://${UH}/${UB}/${base}.zarr#mode=${mode},s3" + echo "flags: $flags" # Dumping everything causes timeout so dump a single var - ${NCDUMP} -v "/group_with_dims/var2D" $flags $url > tmp_${base}_${zext}.cdl + ${NCDUMP} -v "group_with_dims/var2D" $flags $url > tmp_${base}_${zext}.cdl # Find the proper ref file diff -b ${ISOPATH}/ref_${base}_2d.cdl tmp_${base}_${zext}.cdl set +x From 54102a3cea32b31b2638a7e5afebad3f8f35c738 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Mon, 16 Oct 2023 14:30:43 -0600 Subject: [PATCH 25/77] Update path in s3sdk shell script unit test. --- unit_test/run_s3sdk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_test/run_s3sdk.sh b/unit_test/run_s3sdk.sh index 4589bcb5f6..18073e4570 100755 --- a/unit_test/run_s3sdk.sh +++ b/unit_test/run_s3sdk.sh @@ -14,7 +14,7 @@ isolate "testdir_uts3sdk" # Create an isolation path for S3; build on the isolation directory S3ISODIR="$ISODIR" S3ISOPATH="/${S3TESTSUBTREE}" -S3ISOPATH="${S3ISOPATH}/$S3ISODIR" +S3ISOPATH="/${S3ISOPATH}/$S3ISODIR" test_cleanup() { ${CMD} ${execdir}/../nczarr_test/s3util -u "${URL}" -k "${S3ISOPATH}" clear From 523c648e5d388f2528661d0c71f0621e3810c13d Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Mon, 23 Oct 2023 10:20:29 +0100 Subject: [PATCH 26/77] CMake: Don't add uninstall target and CPack config if not top-level Fixes #2597 --- CMakeLists.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0191f1454c..ac0326d119 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,7 @@ # Set Project Properties ################################## -#Minimum required CMake Version -cmake_minimum_required(VERSION 3.12.0) -# CMake 3.12: Use libraries specified in CMAKE_REQUIRED_LIBRARIES for check include macros +cmake_minimum_required(VERSION 3.21.0) #Project Name project(netCDF @@ -2789,5 +2787,6 @@ install( #### # CPack inclusion must come last. -# INCLUDE(CPack) -INCLUDE(CMakeInstallation.cmake) +if (PROJECT_IS_TOP_LEVEL) + include(CMakeInstallation.cmake) +endif() From bce53cae086866bb82cc9354c4716ddb7bb8be5d Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:15:25 -0600 Subject: [PATCH 27/77] Add first script to try to embed cdash scripts into CI. --- .github/workflows/run_tests_cdash.yml | 358 ++++++++++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 .github/workflows/run_tests_cdash.yml diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml new file mode 100644 index 0000000000..ce79c421a8 --- /dev/null +++ b/.github/workflows/run_tests_cdash.yml @@ -0,0 +1,358 @@ +### +# Build hdf5 dependencies and cache them in a combined directory. +### + +name: Run Ubuntu/Linux netCDF Tests with CDash + +on: [pull_request, workflow_dispatch] + +jobs: + + build-deps-cdash: + + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] + + steps: + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev doxygen openssl + + ### + # Installing libhdf5 + ### + - name: Cache libhdf5-${{ matrix.hdf5 }} + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + + - name: Build libhdf5-${{ matrix.hdf5 }} + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + set -x + + wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2 + tar -jxf hdf-4.2.15.tar.bz2 + pushd hdf-4.2.15 + ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-hdf4-xdr + make -j + make install -j + popd + + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2 + tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2 + pushd hdf5-${{ matrix.hdf5 }} + ./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} --enable-hl --with-szlib + make -j + make install -j + popd + + + build-deps-parallel: + + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [ 1.14.0 ] + + steps: + + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev mpich libmpich-dev + + ### + # Installing libhdf5 + ### + - name: Cache libhdf5-parallel-${{ matrix.hdf5 }} + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-parallel-${{ runner.os }}-${{ matrix.hdf5 }} + + + - name: Build libhdf5-${{ matrix.hdf5 }}-pnetcdf-1.12.3 + if: steps.cache-hdf5.outputs.cache-hit != 'true' + run: | + set -x + + wget https://support.hdfgroup.org/ftp/HDF/releases/HDF4.2.15/src/hdf-4.2.15.tar.bz2 + tar -jxf hdf-4.2.15.tar.bz2 + pushd hdf-4.2.15 + CC=mpicc ./configure --prefix=${HOME}/environments/${{ matrix.hdf5 }} --disable-static --enable-shared --disable-fortran --disable-netcdf --with-szlib --enable-parallel --enable-hdf4-xdr + make -j + make install -j + popd + + wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$(echo ${{ matrix.hdf5 }} | cut -d. -f 1,2)/hdf5-${{ matrix.hdf5 }}/src/hdf5-${{ matrix.hdf5 }}.tar.bz2 + tar -jxf hdf5-${{ matrix.hdf5 }}.tar.bz2 + pushd hdf5-${{ matrix.hdf5 }} + CC=mpicc ./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} --enable-hl --with-szlib --enable-parallel + make -j + make install -j + popd + wget https://parallel-netcdf.github.io/Release/pnetcdf-1.12.3.tar.gz + tar -zxf pnetcdf-1.12.3.tar.gz + pushd pnetcdf-1.12.3 + CC=mpicc ./configure --disable-static --enable-shared --prefix=${HOME}/environments/${{ matrix.hdf5 }} + make -j + make install -j + popd + + + + needs: build-deps-parallel + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [ 1.14.0 ] + steps: + + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev mpich libmpich-dev + + ### + # Set Environmental Variables + ### + + - run: echo "CFLAGS=-I${HOME}/environments/${{ matrix.hdf5 }}/include" >> $GITHUB_ENV + - run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + + - name: Fetch HDF Cache + id: cache-hdf + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-parallel-${{ runner.os }}-${{ matrix.hdf5 }} + + - name: Check Cache + shell: bash -l {0} + run: ls ${HOME}/environments && ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + + + ### + # Configure and build + ### + + - name: Run autoconf + shell: bash -l {0} + run: autoreconf -if + + - name: Configure + shell: bash -l {0} + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-parallel-tests --enable-pnetcdf + if: ${{ success() }} + + - name: Look at config.log if error + shell: bash -l {0} + run: cat config.log + if: ${{ failure() }} + + - name: Print Summary + shell: bash -l {0} + run: cat libnetcdf.settings + + - name: Build Library and Utilities + shell: bash -l {0} + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc make -j + if: ${{ success() }} + + - name: Build Tests + shell: bash -l {0} + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc make check TESTS="" -j + if: ${{ success() }} + + - name: Run Tests + shell: bash -l {0} + run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc make check -j + if: ${{ success() }} + + + + + needs: [ nc-cmake-tests-oneoff-cdash, nc-ac-tests-oneoff-cdash, nc-cmake-tests-oneoff-parallel, nc-ac-tests-oneoff-parallel ] + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [1.10.8, 1.12.2, 1.14.0 ] + use_nc4: [ nc3, nc4 ] + use_dap: [ dap_off, dap_on ] + use_nczarr: [ nczarr_off, nczarr_on ] + steps: + + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev + + ### + # Set Environmental Variables + ### + + - run: echo "CFLAGS=-I${HOME}/environments/${{ matrix.hdf5 }}/include" >> $GITHUB_ENV + - run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + - run: | + echo "ENABLE_HDF5=--disable-hdf5" >> $GITHUB_ENV + if: matrix.use_nc4 == 'nc3' + - run: | + echo "ENABLE_HDF5=--enable-hdf5" >> $GITHUB_ENV + if: matrix.use_nc4 == 'nc4' + - run: echo "ENABLE_DAP=--disable-dap" >> $GITHUB_ENV + if: matrix.use_dap == 'dap_off' + - run: echo "ENABLE_DAP=--enable-dap" >> $GITHUB_ENV + if: matrix.use_dap == 'dap_on' + - run: echo "ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV + if: matrix.use_nczarr == 'nczarr_off' + - run: echo "ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV + if: matrix.use_nczarr == 'nczarr_on' + + ### + # Fetch Cache + ### + + - name: Fetch HDF Cache + id: cache-hdf + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + - name: Check Cache + shell: bash -l {0} + run: ls ${HOME}/environments && ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + + ### + # Configure and build + ### + + - name: Run autoconf + shell: bash -l {0} + run: autoreconf -if + + - name: Configure + shell: bash -l {0} + run: | + current_directory="$(pwd)" + mkdir ../build + cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} "${current_directory}/configure" ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR} + if: ${{ success() }} + + - name: Look at config.log if error + shell: bash -l {0} + run: cd ../build && cat config.log + if: ${{ failure() }} + + - name: Print Summary + shell: bash -l {0} + run: cd ../build && cat libnetcdf.settings + + - name: Build Library and Utilities + shell: bash -l {0} + run: | + cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j + if: ${{ success() }} + + - name: Build Tests + shell: bash -l {0} + run: | + cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j + if: ${{ success() }} + + - name: Run Tests + shell: bash -l {0} + run: | + cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j + if: ${{ success() }} + + nc-cmake: + + needs: build-deps-cdash + runs-on: ubuntu-latest + + strategy: + matrix: + hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] + use_nc4: [ nc3, nc4 ] + use_dap: [ dap_off, dap_on ] + use_nczarr: [ nczarr_off, nczarr_on ] + steps: + + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev + + ### + # Set Environmental Variables + ### + + - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV + - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + - run: | + echo "ENABLE_HDF5=OFF" >> $GITHUB_ENV + if: matrix.use_nc4 == 'nc3' + - run: | + echo "ENABLE_HDF5=ON" >> $GITHUB_ENV + if: matrix.use_nc4 == 'nc4' + - run: echo "ENABLE_DAP=OFF" >> $GITHUB_ENV + if: matrix.use_dap == 'dap_off' + - run: echo "ENABLE_DAP=ON" >> $GITHUB_ENV + if: matrix.use_dap == 'dap_on' + - run: echo "ENABLE_NCZARR=OFF" >> $GITHUB_ENV + if: matrix.use_nczarr == 'nczarr_off' + - run: echo "ENABLE_NCZARR=ON" >> $GITHUB_ENV + if: matrix.use_nczarr == 'nczarr_on' + - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV + + ### + # Fetch Cache + ### + + - name: Fetch HDF Cache + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + - name: Check Cache + shell: bash -l {0} + run: ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + + ### + # Run CTest Serial Script + ### + - name: Run ctest serial script + shell: bash -l {0} + run: | + mkdir build + cd build + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -V -S ../ctest_scripts/ctest_serial.ctest + + - name: Verbose Output if CTest Failure + shell: bash -l {0} + run: | + cd build + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 --rerun-failed --output-on-failure -VV + if: ${{ failure() }} From df5261ce0cb71d68788c7f31f99f2d087303c32b Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:17:13 -0600 Subject: [PATCH 28/77] Correct issue with file. --- .github/workflows/run_tests_cdash.yml | 230 +------------------------- 1 file changed, 1 insertion(+), 229 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index ce79c421a8..8b19810c0a 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -110,236 +110,8 @@ jobs: make -j make install -j popd - - - - needs: build-deps-parallel - runs-on: ubuntu-latest - - strategy: - matrix: - hdf5: [ 1.14.0 ] - steps: - - - uses: actions/checkout@v3 - - - name: Install System dependencies - shell: bash -l {0} - run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev mpich libmpich-dev - - ### - # Set Environmental Variables - ### - - - run: echo "CFLAGS=-I${HOME}/environments/${{ matrix.hdf5 }}/include" >> $GITHUB_ENV - - run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - - - name: Fetch HDF Cache - id: cache-hdf - uses: actions/cache@v3 - with: - path: ~/environments/${{ matrix.hdf5 }} - key: hdf5-parallel-${{ runner.os }}-${{ matrix.hdf5 }} - - - name: Check Cache - shell: bash -l {0} - run: ls ${HOME}/environments && ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib - - - ### - # Configure and build - ### - - - name: Run autoconf - shell: bash -l {0} - run: autoreconf -if - - - name: Configure - shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --enable-parallel-tests --enable-pnetcdf - if: ${{ success() }} - - - name: Look at config.log if error - shell: bash -l {0} - run: cat config.log - if: ${{ failure() }} - - - name: Print Summary - shell: bash -l {0} - run: cat libnetcdf.settings - - - name: Build Library and Utilities - shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc make -j - if: ${{ success() }} - - - name: Build Tests - shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc make check TESTS="" -j - if: ${{ success() }} - - - name: Run Tests - shell: bash -l {0} - run: CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} CC=mpicc make check -j - if: ${{ success() }} - - - - - needs: [ nc-cmake-tests-oneoff-cdash, nc-ac-tests-oneoff-cdash, nc-cmake-tests-oneoff-parallel, nc-ac-tests-oneoff-parallel ] - runs-on: ubuntu-latest - - strategy: - matrix: - hdf5: [1.10.8, 1.12.2, 1.14.0 ] - use_nc4: [ nc3, nc4 ] - use_dap: [ dap_off, dap_on ] - use_nczarr: [ nczarr_off, nczarr_on ] - steps: - - - uses: actions/checkout@v3 - - - name: Install System dependencies - shell: bash -l {0} - run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev - - ### - # Set Environmental Variables - ### - - - run: echo "CFLAGS=-I${HOME}/environments/${{ matrix.hdf5 }}/include" >> $GITHUB_ENV - - run: echo "LDFLAGS=-L${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - - run: | - echo "ENABLE_HDF5=--disable-hdf5" >> $GITHUB_ENV - if: matrix.use_nc4 == 'nc3' - - run: | - echo "ENABLE_HDF5=--enable-hdf5" >> $GITHUB_ENV - if: matrix.use_nc4 == 'nc4' - - run: echo "ENABLE_DAP=--disable-dap" >> $GITHUB_ENV - if: matrix.use_dap == 'dap_off' - - run: echo "ENABLE_DAP=--enable-dap" >> $GITHUB_ENV - if: matrix.use_dap == 'dap_on' - - run: echo "ENABLE_NCZARR=--disable-nczarr" >> $GITHUB_ENV - if: matrix.use_nczarr == 'nczarr_off' - - run: echo "ENABLE_NCZARR=--enable-nczarr" >> $GITHUB_ENV - if: matrix.use_nczarr == 'nczarr_on' - - ### - # Fetch Cache - ### - - - name: Fetch HDF Cache - id: cache-hdf - uses: actions/cache@v3 - with: - path: ~/environments/${{ matrix.hdf5 }} - key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} - - - name: Check Cache - shell: bash -l {0} - run: ls ${HOME}/environments && ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib - - ### - # Configure and build - ### - - - name: Run autoconf - shell: bash -l {0} - run: autoreconf -if - - - name: Configure - shell: bash -l {0} - run: | - current_directory="$(pwd)" - mkdir ../build - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} "${current_directory}/configure" ${ENABLE_HDF5} ${ENABLE_DAP} ${ENABLE_NCZARR} - if: ${{ success() }} - - - name: Look at config.log if error - shell: bash -l {0} - run: cd ../build && cat config.log - if: ${{ failure() }} - - - name: Print Summary - shell: bash -l {0} - run: cd ../build && cat libnetcdf.settings - - - name: Build Library and Utilities - shell: bash -l {0} - run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make -j - if: ${{ success() }} - - - name: Build Tests - shell: bash -l {0} - run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check TESTS="" -j - if: ${{ success() }} - - - name: Run Tests - shell: bash -l {0} - run: | - cd ../build && CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} LD_LIBRARY_PATH=${LD_LIBRARY_PATH} make check -j - if: ${{ success() }} - - nc-cmake: - - needs: build-deps-cdash - runs-on: ubuntu-latest - - strategy: - matrix: - hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] - use_nc4: [ nc3, nc4 ] - use_dap: [ dap_off, dap_on ] - use_nczarr: [ nczarr_off, nczarr_on ] - steps: - - - uses: actions/checkout@v3 - - - name: Install System dependencies - shell: bash -l {0} - run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev - - ### - # Set Environmental Variables - ### - - - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV - - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - - run: | - echo "ENABLE_HDF5=OFF" >> $GITHUB_ENV - if: matrix.use_nc4 == 'nc3' - - run: | - echo "ENABLE_HDF5=ON" >> $GITHUB_ENV - if: matrix.use_nc4 == 'nc4' - - run: echo "ENABLE_DAP=OFF" >> $GITHUB_ENV - if: matrix.use_dap == 'dap_off' - - run: echo "ENABLE_DAP=ON" >> $GITHUB_ENV - if: matrix.use_dap == 'dap_on' - - run: echo "ENABLE_NCZARR=OFF" >> $GITHUB_ENV - if: matrix.use_nczarr == 'nczarr_off' - - run: echo "ENABLE_NCZARR=ON" >> $GITHUB_ENV - if: matrix.use_nczarr == 'nczarr_on' - - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV - - ### - # Fetch Cache - ### - - - name: Fetch HDF Cache - id: cache-hdf5 - uses: actions/cache@v3 - with: - path: ~/environments/${{ matrix.hdf5 }} - key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} - - - name: Check Cache - shell: bash -l {0} - run: ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + ### # Run CTest Serial Script ### From b41c33805aba64ce636f01e10b9db135105b2f00 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:18:58 -0600 Subject: [PATCH 29/77] Update cdash script file. --- .github/workflows/run_tests_cdash.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 8b19810c0a..7d11008b8a 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -2,7 +2,7 @@ # Build hdf5 dependencies and cache them in a combined directory. ### -name: Run Ubuntu/Linux netCDF Tests with CDash +name: Run CDash Ubuntu/Linux netCDF Tests on: [pull_request, workflow_dispatch] @@ -111,7 +111,7 @@ jobs: make install -j popd - + ### # Run CTest Serial Script ### From 7d9baced0eafc5429778a3160bec5118c82c2632 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:28:51 -0600 Subject: [PATCH 30/77] Correct issue with test file. --- .github/workflows/run_tests_cdash.yml | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 7d11008b8a..7da75455a4 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -115,6 +115,60 @@ jobs: ### # Run CTest Serial Script ### + nc-ctest-serial: + needs: build-deps-cdash + runs-on: ubuntu-latest + environment: CDashCI + + strategy: + matrix: + hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] + use_nc4: [ nc3, nc4 ] + use_dap: [ dap_off, dap_on ] + use_nczarr: [ nczarr_off, nczarr_on ] + steps: + - uses: actions/checkout@v3 + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev + + ### + # Set Environmental Variables + ### + + - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV + - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + - run: | + echo "ENABLE_HDF5=OFF" >> $GITHUB_ENV + if: matrix.use_nc4 == 'nc3' + - run: | + echo "ENABLE_HDF5=ON" >> $GITHUB_ENV + if: matrix.use_nc4 == 'nc4' + - run: echo "ENABLE_DAP=OFF" >> $GITHUB_ENV + if: matrix.use_dap == 'dap_off' + - run: echo "ENABLE_DAP=ON" >> $GITHUB_ENV + if: matrix.use_dap == 'dap_on' + - run: echo "ENABLE_NCZARR=OFF" >> $GITHUB_ENV + if: matrix.use_nczarr == 'nczarr_off' + - run: echo "ENABLE_NCZARR=ON" >> $GITHUB_ENV + if: matrix.use_nczarr == 'nczarr_on' + - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV + + ### + # Fetch Cache + ### + + - name: Fetch HDF Cache + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + - name: Check Cache + shell: bash -l {0} + run: ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib - name: Run ctest serial script shell: bash -l {0} run: | From 7a28aefa5264d6c5d0725ee53ec7a0a8b065e74b Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:32:31 -0600 Subject: [PATCH 31/77] Reduce cdash/ctest-based script matrix. --- .github/workflows/run_tests_cdash.yml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 7da75455a4..5e922803cf 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -123,9 +123,6 @@ jobs: strategy: matrix: hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] - use_nc4: [ nc3, nc4 ] - use_dap: [ dap_off, dap_on ] - use_nczarr: [ nczarr_off, nczarr_on ] steps: - uses: actions/checkout@v3 @@ -139,20 +136,6 @@ jobs: - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - - run: | - echo "ENABLE_HDF5=OFF" >> $GITHUB_ENV - if: matrix.use_nc4 == 'nc3' - - run: | - echo "ENABLE_HDF5=ON" >> $GITHUB_ENV - if: matrix.use_nc4 == 'nc4' - - run: echo "ENABLE_DAP=OFF" >> $GITHUB_ENV - if: matrix.use_dap == 'dap_off' - - run: echo "ENABLE_DAP=ON" >> $GITHUB_ENV - if: matrix.use_dap == 'dap_on' - - run: echo "ENABLE_NCZARR=OFF" >> $GITHUB_ENV - if: matrix.use_nczarr == 'nczarr_off' - - run: echo "ENABLE_NCZARR=ON" >> $GITHUB_ENV - if: matrix.use_nczarr == 'nczarr_on' - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV ### @@ -174,7 +157,7 @@ jobs: run: | mkdir build cd build - LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -V -S ../ctest_scripts/ctest_serial.ctest + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 -V -S ../ctest_scripts/ctest_serial.ctest - name: Verbose Output if CTest Failure shell: bash -l {0} From 7ab20391d2b10dc7bb9c596b7ce02d47713ee321 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:39:11 -0600 Subject: [PATCH 32/77] Add coverage testing to a specific test. --- ctest_scripts/ctest_serial_coverage.ctest | 48 +++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 ctest_scripts/ctest_serial_coverage.ctest diff --git a/ctest_scripts/ctest_serial_coverage.ctest b/ctest_scripts/ctest_serial_coverage.ctest new file mode 100644 index 0000000000..522ca0b8c8 --- /dev/null +++ b/ctest_scripts/ctest_serial_coverage.ctest @@ -0,0 +1,48 @@ +### +# Standard CTest Script for testing netCDF. +# Requires a CDash Token. +# +# Set the CDASH_TOKEN environmental variable. +# +### + +SET(CTEST_SOURCE_DIRECTORY "..") +SET(CTEST_BINARY_DIRECTORY ".") + +set(CDASH_TOKEN $ENV{CDASH_TOKEN}) +MESSAGE("Using cdash token: ${CDASH_TOKEN}") + + +SITE_NAME(local_site_name) +set(CTEST_SITE ${local_site_name}) + +set(CTEST_BUILD_CONFIGURATION "Profiling") +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +set(CTEST_BUILD_NAME "default") +set(CTEST_BUILD_CONFIGURATION "Profiling") +set(CTEST_DROP_METHOD "https") +set(CTEST_DROP_SITE_CDASH TRUE) +set(CTEST_PROJECT_NAME netcdf-c) + +find_program(CMAKE_COMMAND cmake) +find_program(CTEST_GIT_COMMAND NAMES git) +find_program(CTEST_COVERAGE_COMMAND NAMES gcov) +find_program(CTEST_MEMORYCHECK_COMMAND NAMES valgrind) + +set(CTEST_BUILD_OPTIONS "-DENABLE_COVERAGE_TESTS=TRUE -DENABLE_ERANGE_FILL=TRUE -DENABLE_LOGGING=TRUE -DENABLE_BYTERANGE=TRUE -DENABLE_LARGE_FILE_TESTS=FALSE") + + +set(CTEST_DROP_METHOD https) +set(CTEST_DROP_SITE "cdash.unidata.ucar.edu:443") +set(CTEST_DROP_LOCATION "/submit.php?project=netcdf-c") + +set(CTEST_CONFIGURE_COMMAND "${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE:STRING=${CTEST_BUILD_CONFIGURATION} ${CTEST_BUILD_OPTIONS} ${CTEST_SOURCE_DIRECTORY}") + +ctest_start("Experimental") +ctest_configure() +ctest_build() +ctest_test() +ctest_coverage() +if (NOT "${CDASH_TOKEN}" STREQUAL "") + ctest_submit(HTTPHEADER "Authorization: Bearer ${CDASH_TOKEN}") +endif() From 794ff5adad6cd2233645b8393d83fbb85e1c2ce5 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:39:44 -0600 Subject: [PATCH 33/77] Turn off coverage testing in ctest_serial.ctest. --- ctest_scripts/ctest_serial.ctest | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ctest_scripts/ctest_serial.ctest b/ctest_scripts/ctest_serial.ctest index 522ca0b8c8..cdcb2912df 100644 --- a/ctest_scripts/ctest_serial.ctest +++ b/ctest_scripts/ctest_serial.ctest @@ -29,7 +29,7 @@ find_program(CTEST_GIT_COMMAND NAMES git) find_program(CTEST_COVERAGE_COMMAND NAMES gcov) find_program(CTEST_MEMORYCHECK_COMMAND NAMES valgrind) -set(CTEST_BUILD_OPTIONS "-DENABLE_COVERAGE_TESTS=TRUE -DENABLE_ERANGE_FILL=TRUE -DENABLE_LOGGING=TRUE -DENABLE_BYTERANGE=TRUE -DENABLE_LARGE_FILE_TESTS=FALSE") +set(CTEST_BUILD_OPTIONS "-DENABLE_COVERAGE_TESTS=FALSE -DENABLE_ERANGE_FILL=TRUE -DENABLE_LOGGING=TRUE -DENABLE_BYTERANGE=TRUE -DENABLE_LARGE_FILE_TESTS=FALSE") set(CTEST_DROP_METHOD https) @@ -42,7 +42,6 @@ ctest_start("Experimental") ctest_configure() ctest_build() ctest_test() -ctest_coverage() if (NOT "${CDASH_TOKEN}" STREQUAL "") ctest_submit(HTTPHEADER "Authorization: Bearer ${CDASH_TOKEN}") endif() From 4e7a864c2371a43cc5b66c9f2d489aad3930ea95 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:48:51 -0600 Subject: [PATCH 34/77] Working on automating cdash access on github. --- .github/workflows/run_tests_cdash.yml | 3 +++ ctest_scripts/ctest_serial.ctest | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 5e922803cf..27042ee01e 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -152,7 +152,10 @@ jobs: - name: Check Cache shell: bash -l {0} run: ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + - name: Run ctest serial script + with: # Access the CDash TOken + CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} shell: bash -l {0} run: | mkdir build diff --git a/ctest_scripts/ctest_serial.ctest b/ctest_scripts/ctest_serial.ctest index cdcb2912df..9a5d60cc5d 100644 --- a/ctest_scripts/ctest_serial.ctest +++ b/ctest_scripts/ctest_serial.ctest @@ -10,6 +10,13 @@ SET(CTEST_SOURCE_DIRECTORY "..") SET(CTEST_BINARY_DIRECTORY ".") set(CDASH_TOKEN $ENV{CDASH_TOKEN}) + +IF (CDASH_TOKEN) + MESSAGE("CDASH TOKEN FOUND") +ELSE (CDASH_TOKEN) + MESSAGE("NO CDASH TOKEN FOUND") +ENDIF (CDASH_TOKEN) + MESSAGE("Using cdash token: ${CDASH_TOKEN}") From 88ecadfdb0476336777ab97ae4dcee5221253f52 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:51:20 -0600 Subject: [PATCH 35/77] Adjust syntax. --- .github/workflows/run_tests_cdash.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 27042ee01e..bd7c8e3aa4 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -123,6 +123,7 @@ jobs: strategy: matrix: hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] + steps: - uses: actions/checkout@v3 @@ -154,9 +155,9 @@ jobs: run: ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib - name: Run ctest serial script - with: # Access the CDash TOken - CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} shell: bash -l {0} + env: + CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} run: | mkdir build cd build From 74df474ac71a22834e0405530da8bf5e72ccf166 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 10:55:57 -0600 Subject: [PATCH 36/77] Adjust github action config file syntax. --- .github/workflows/run_tests_cdash.yml | 2 +- ctest_scripts/ctest_serial.ctest | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index bd7c8e3aa4..2b553de846 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -156,7 +156,7 @@ jobs: - name: Run ctest serial script shell: bash -l {0} - env: + with: CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} run: | mkdir build diff --git a/ctest_scripts/ctest_serial.ctest b/ctest_scripts/ctest_serial.ctest index 9a5d60cc5d..71a6eb6969 100644 --- a/ctest_scripts/ctest_serial.ctest +++ b/ctest_scripts/ctest_serial.ctest @@ -17,9 +17,6 @@ ELSE (CDASH_TOKEN) MESSAGE("NO CDASH TOKEN FOUND") ENDIF (CDASH_TOKEN) -MESSAGE("Using cdash token: ${CDASH_TOKEN}") - - SITE_NAME(local_site_name) set(CTEST_SITE ${local_site_name}) From 461569509157d52ac517f7e5f1bb23d44fbd0ba7 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 11:00:17 -0600 Subject: [PATCH 37/77] Playing around trying to nail down syntax for accessing secrets in Github Actions. --- .github/workflows/run_tests_cdash.yml | 80 +++++++++++++-------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 2b553de846..f11f13af3d 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -125,47 +125,47 @@ jobs: hdf5: [ 1.10.8, 1.12.2, 1.14.0 ] steps: - - uses: actions/checkout@v3 - - - name: Install System dependencies - shell: bash -l {0} - run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev - - ### - # Set Environmental Variables - ### + - uses: actions/checkout@v3 + with: + CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} + + - name: Install System dependencies + shell: bash -l {0} + run: sudo apt update && sudo apt install -y libaec-dev zlib1g-dev automake autoconf libcurl4-openssl-dev libjpeg-dev wget curl bzip2 m4 flex bison cmake libzip-dev - - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV - - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV - - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV - - ### - # Fetch Cache - ### - - - name: Fetch HDF Cache - id: cache-hdf5 - uses: actions/cache@v3 - with: - path: ~/environments/${{ matrix.hdf5 }} - key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + ### + # Set Environmental Variables + ### - - name: Check Cache - shell: bash -l {0} - run: ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + - run: echo "CMAKE_PREFIX_PATH=${HOME}/environments/${{ matrix.hdf5 }}/" >> $GITHUB_ENV + - run: echo "LD_LIBRARY_PATH=${HOME}/environments/${{ matrix.hdf5 }}/lib" >> $GITHUB_ENV + - run: echo "CTEST_OUTPUT_ON_FAILURE=1" >> $GITHUB_ENV - - name: Run ctest serial script - shell: bash -l {0} - with: - CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} - run: | - mkdir build - cd build - LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 -V -S ../ctest_scripts/ctest_serial.ctest + ### + # Fetch Cache + ### - - name: Verbose Output if CTest Failure - shell: bash -l {0} - run: | - cd build - LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 --rerun-failed --output-on-failure -VV - if: ${{ failure() }} + - name: Fetch HDF Cache + id: cache-hdf5 + uses: actions/cache@v3 + with: + path: ~/environments/${{ matrix.hdf5 }} + key: hdf5-${{ runner.os }}-${{ matrix.hdf5 }} + + - name: Check Cache + shell: bash -l {0} + run: ls ${HOME}/environments/${{ matrix.hdf5 }} && ls ${HOME}/environments/${{ matrix.hdf5}}/lib + + - name: Run ctest serial script + shell: bash -l {0} + run: | + mkdir build + cd build + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 -V -S ../ctest_scripts/ctest_serial.ctest + + - name: Verbose Output if CTest Failure + shell: bash -l {0} + run: | + cd build + LD_LIBRARY_PATH=${LD_LIBRARY_PATH} ctest -j 12 --rerun-failed --output-on-failure -VV + if: ${{ failure() }} From dcad99082dbdc113694c7bf8467414a858d53abd Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 11:13:04 -0600 Subject: [PATCH 38/77] Remove sensitive info being sent to standard output when using ctest scripts. --- CMakeLists.txt | 4 ++-- ctest_scripts/ctest_mpicc.ctest | 6 +++++- ctest_scripts/ctest_serial_coverage.ctest | 6 +++++- ctest_scripts/ctest_serial_dap.ctest | 6 +++++- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0191f1454c..6fa02dcd6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -267,8 +267,8 @@ IF(CMAKE_COMPILER_IS_GNUCC OR APPLE) # Coverage tests need to have optimization turned off. IF(ENABLE_COVERAGE_TESTS) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") + SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage") + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs") MESSAGE(STATUS "Coverage Tests: On.") ENDIF() diff --git a/ctest_scripts/ctest_mpicc.ctest b/ctest_scripts/ctest_mpicc.ctest index 9bca2038d8..617ea49a22 100644 --- a/ctest_scripts/ctest_mpicc.ctest +++ b/ctest_scripts/ctest_mpicc.ctest @@ -10,7 +10,11 @@ SET(CTEST_SOURCE_DIRECTORY "..") SET(CTEST_BINARY_DIRECTORY ".") set(CDASH_TOKEN $ENV{CDASH_TOKEN}) -MESSAGE("Using cdash token: ${CDASH_TOKEN}") +IF (CDASH_TOKEN) + MESSAGE("CDASH TOKEN FOUND") +ELSE (CDASH_TOKEN) + MESSAGE("NO CDASH TOKEN FOUND") +ENDIF (CDASH_TOKEN) SITE_NAME(local_site_name) diff --git a/ctest_scripts/ctest_serial_coverage.ctest b/ctest_scripts/ctest_serial_coverage.ctest index 522ca0b8c8..9b8f90a201 100644 --- a/ctest_scripts/ctest_serial_coverage.ctest +++ b/ctest_scripts/ctest_serial_coverage.ctest @@ -10,7 +10,11 @@ SET(CTEST_SOURCE_DIRECTORY "..") SET(CTEST_BINARY_DIRECTORY ".") set(CDASH_TOKEN $ENV{CDASH_TOKEN}) -MESSAGE("Using cdash token: ${CDASH_TOKEN}") +IF (CDASH_TOKEN) + MESSAGE("CDASH TOKEN FOUND") +ELSE (CDASH_TOKEN) + MESSAGE("NO CDASH TOKEN FOUND") +ENDIF (CDASH_TOKEN) SITE_NAME(local_site_name) diff --git a/ctest_scripts/ctest_serial_dap.ctest b/ctest_scripts/ctest_serial_dap.ctest index 9ea3333459..80fe31264d 100644 --- a/ctest_scripts/ctest_serial_dap.ctest +++ b/ctest_scripts/ctest_serial_dap.ctest @@ -10,7 +10,11 @@ SET(CTEST_SOURCE_DIRECTORY "..") SET(CTEST_BINARY_DIRECTORY ".") set(CDASH_TOKEN $ENV{CDASH_TOKEN}) -MESSAGE("Using cdash token: ${CDASH_TOKEN}") +IF (CDASH_TOKEN) + MESSAGE("CDASH TOKEN FOUND") +ELSE (CDASH_TOKEN) + MESSAGE("NO CDASH TOKEN FOUND") +ENDIF (CDASH_TOKEN) SITE_NAME(local_site_name) From ee9c9d91aa1ab54968ec01d00899d0aad82b2c21 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 11:25:02 -0600 Subject: [PATCH 39/77] Try a different syntax. --- .github/workflows/run_tests_cdash.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index f11f13af3d..5caf899421 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -128,6 +128,8 @@ jobs: - uses: actions/checkout@v3 with: CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} + env: + CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} - name: Install System dependencies shell: bash -l {0} @@ -158,6 +160,8 @@ jobs: - name: Run ctest serial script shell: bash -l {0} + env: + CDASH_TOKEN: ${{ secrets.CDASH_TOKEN }} run: | mkdir build cd build From 4483760d3267d5da944c4080e9da2ad43e8f58da Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 11:30:56 -0600 Subject: [PATCH 40/77] Adjust workflow secrets syntax. --- .github/workflows/run_tests_cdash.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 5caf899421..927f10a657 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -5,6 +5,11 @@ name: Run CDash Ubuntu/Linux netCDF Tests on: [pull_request, workflow_dispatch] + workflow_call: + secrets: + CDASH_TOKEN: + description: 'A token required to access the cdash dashboard' + required: true jobs: From 127830e49f32bb61dcadcc185bb818b1444b6d14 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 11:42:49 -0600 Subject: [PATCH 41/77] Shot in the dark --- .github/workflows/run_tests_cdash.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 927f10a657..849427af6d 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -4,12 +4,16 @@ name: Run CDash Ubuntu/Linux netCDF Tests -on: [pull_request, workflow_dispatch] +on: + workflow_dispatch: + pull_request: workflow_call: secrets: - CDASH_TOKEN: - description: 'A token required to access the cdash dashboard' - required: true + CDASH_TOKEN: + description: 'A token required to access the cdash dashboard' + required: true + + jobs: From 3e341bd6e1c93bd913fc7af046c2add4192455a9 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 11:46:21 -0600 Subject: [PATCH 42/77] Turning off cdash integration for the moment, but merging PR to include other changes/updates. --- .github/workflows/run_tests_cdash.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 849427af6d..57053172cd 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -4,15 +4,8 @@ name: Run CDash Ubuntu/Linux netCDF Tests -on: - workflow_dispatch: - pull_request: - workflow_call: - secrets: - CDASH_TOKEN: - description: 'A token required to access the cdash dashboard' - required: true - +on: workflow_dispatch + jobs: From bbcbab7a26a787870cd85aa306f4bd8f70c85cb9 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 11:48:29 -0600 Subject: [PATCH 43/77] Clarify name of a Github Action. --- .github/workflows/run_tests_win_mingw.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index f48e81e7fa..7bc6cde76a 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -4,7 +4,7 @@ # for information related to github runners. ### -name: Run MSYS2, MinGW64-based Tests +name: Run MSYS2, MinGW64-based Tests (Not Visual Studio) env: CPPFLAGS: "-D_BSD_SOURCE" From 347b2504bfddc8105c36d41b489977066f649fa6 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 13:15:12 -0600 Subject: [PATCH 44/77] Undo previous change. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fa02dcd6a..59bb846328 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -267,7 +267,8 @@ IF(CMAKE_COMPILER_IS_GNUCC OR APPLE) # Coverage tests need to have optimization turned off. IF(ENABLE_COVERAGE_TESTS) - SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage") SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs") MESSAGE(STATUS "Coverage Tests: On.") ENDIF() From d4766297041d3727f063b4f25040b20808ea7169 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 24 Oct 2023 13:16:11 -0600 Subject: [PATCH 45/77] Replace accidental flag removal --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59bb846328..29c505aadd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -269,7 +269,7 @@ IF(CMAKE_COMPILER_IS_GNUCC OR APPLE) IF(ENABLE_COVERAGE_TESTS) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -coverage -fprofile-arcs -ftest-coverage") - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs") + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage") MESSAGE(STATUS "Coverage Tests: On.") ENDIF() From b41b79a8e57a73f27ba9754d62c047425ac94d1f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 25 Oct 2023 11:50:07 +0100 Subject: [PATCH 46/77] CMake: Add backport of `PROJECT_IS_TOP_LEVEL` for CMake <3.21 --- CMakeLists.txt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac0326d119..b070048eff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ # Set Project Properties ################################## -cmake_minimum_required(VERSION 3.21.0) +cmake_minimum_required(VERSION 3.12.0) #Project Name project(netCDF @@ -16,6 +16,14 @@ DESCRIPTION "NetCDF is a set of software libraries and machine-independent data ) set(PACKAGE "netCDF" CACHE STRING "") +# Backport of built-in `PROJECT_IS_TOP_LEVEL` from CMake 3.21 +if (NOT DEFINED NETCDF_IS_TOP_LEVEL) + set(NETCDF_IS_TOP_LEVEL OFF) + if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + set(NETCDF_IS_TOP_LEVEL ON) + endif () +endif () + ##### # Version Info: # @@ -2787,6 +2795,8 @@ install( #### # CPack inclusion must come last. -if (PROJECT_IS_TOP_LEVEL) +option(NETCDF_PACKAGE "Create netCDF-C package " ${NETCDF_IS_TOP_LEVEL}) + +if (NETCDF_PACKAGE) include(CMakeInstallation.cmake) endif() From 1c0dfaa6c1eb38022650bb690c1eae18434294e6 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Wed, 25 Oct 2023 10:23:26 -0600 Subject: [PATCH 47/77] Added stanza to workflow actions so that pushed changes cancel any running tests in favor of testing the updated codebase. --- .github/workflows/run_tests_cdash.yml | 4 +++- .github/workflows/run_tests_osx.yml | 5 ++++- .github/workflows/run_tests_s3.yml | 4 ++++ .github/workflows/run_tests_ubuntu.yml | 4 ++++ .github/workflows/run_tests_win_cygwin.yml | 4 ++++ .github/workflows/run_tests_win_mingw.yml | 4 ++++ 6 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests_cdash.yml b/.github/workflows/run_tests_cdash.yml index 57053172cd..b8d10ef118 100644 --- a/.github/workflows/run_tests_cdash.yml +++ b/.github/workflows/run_tests_cdash.yml @@ -6,7 +6,9 @@ name: Run CDash Ubuntu/Linux netCDF Tests on: workflow_dispatch - +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true jobs: diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index c887c6b1e6..8f2d9d62af 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -6,9 +6,12 @@ name: Run macOS-based netCDF Tests - on: [pull_request,workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + jobs: build-deps-osx: diff --git a/.github/workflows/run_tests_s3.yml b/.github/workflows/run_tests_s3.yml index 0a1c942460..ddbb353c8e 100644 --- a/.github/workflows/run_tests_s3.yml +++ b/.github/workflows/run_tests_s3.yml @@ -11,6 +11,10 @@ name: Run S3 netCDF Tests (under Ubuntu Linux) on: [workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + jobs: build-deps-serial: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 63afb906c0..64b91bb99a 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -6,6 +6,10 @@ name: Run Ubuntu/Linux netCDF Tests on: [pull_request, workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + jobs: build-deps-serial: diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index bc084a401f..6e124448ed 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -2,6 +2,10 @@ name: Run Cygwin-based tests on: [pull_request,workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + env: SHELLOPTS: igncr CHERE_INVOKING: 1 diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 7bc6cde76a..978275cf6c 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -11,6 +11,10 @@ env: on: [pull_request,workflow_dispatch] +concurrency: + group: ${{ github.workflow}}-${{ github.head_ref }} + cancel-in-progress: true + jobs: build-and-test-autotools: From 763e54f8a11b07bd247221fedac9bdb7492f4fa3 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 26 Oct 2023 16:01:24 +0100 Subject: [PATCH 48/77] Fix most float conversion warnings Fixes ~200 out of ~240 `-Wfloat-conversion` warnings on gcc 13.2 --- examples/C/format.c | 2 +- examples/C/pres_temp_4D_wr.c | 16 ++++++------- examples/C/sfc_pres_temp_wr.c | 16 ++++++------- h5_test/tst_h_atts3.c | 2 +- h5_test/tst_h_atts4.c | 2 +- h5_test/tst_h_compounds.c | 24 ++++++++++---------- h5_test/tst_h_vars.c | 2 +- h5_test/tst_h_vars2.c | 2 +- libhdf5/hdf5internal.c | 2 +- libhdf5/hdf5open.c | 2 +- libhdf5/hdf5var.c | 2 +- nc_test/tst_diskless.c | 8 +++---- nc_test/util.c | 4 ++-- nc_test4/cdm_sea_soundings.c | 6 ++--- nc_test4/tst_atts3.c | 14 ++++++------ nc_test4/tst_chunks2.c | 2 +- nc_test4/tst_compounds.c | 14 ++++++------ nc_test4/tst_compounds2.c | 2 +- nc_test4/tst_compounds3.c | 2 +- nc_test4/tst_converts2.c | 2 +- nc_test4/tst_coords.c | 14 ++++++------ nc_test4/tst_dims.c | 8 +++---- nc_test4/tst_files.c | 2 +- nc_test4/tst_put_vars.c | 2 +- nc_test4/tst_quantize.c | 20 ++++++++--------- nc_test4/tst_unlim_vars.c | 2 +- nc_test4/tst_vars.c | 18 +++++++-------- nc_test4/tst_xplatform2.c | 6 ++--- ncdap_test/t_dap3a.c | 2 +- ncdap_test/test_cvt.c | 2 +- ncdap_test/test_varm3.c | 42 +++++++++++++++++------------------ ncdump/nccopy.c | 7 +++--- ncdump/ncdump.c | 12 +++++----- ncdump/tst_create_files.c | 4 ++-- ncdump/tst_vlen_data.c | 2 +- nczarr_test/test_zchunks2.c | 2 +- 36 files changed, 136 insertions(+), 135 deletions(-) diff --git a/examples/C/format.c b/examples/C/format.c index 291aad65b6..40413dbe3f 100644 --- a/examples/C/format.c +++ b/examples/C/format.c @@ -45,7 +45,7 @@ main() /* Create a bunch of phoney data so we have something to write in the example file. */ for (fp=(float *)temp, i=0; ichunkcache.nelems), &(var->chunkcache.size), &rdcc_w0)) < 0) BAIL(NC_EHDFERR); - var->chunkcache.preemption = rdcc_w0; + var->chunkcache.preemption = (float)rdcc_w0; /* Get the dataset creation properties. */ if ((propid = H5Dget_create_plist(hdf5_var->hdf_datasetid)) < 0) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 0a1273250b..6d37fe9099 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -2391,7 +2391,7 @@ nc_set_var_chunk_cache_ints(int ncid, int varid, int size, int nelems, real_nelems = nelems; if (preemption >= 0) - real_preemption = preemption / 100.; + real_preemption = (float)(preemption / 100.); return NC4_HDF5_set_var_chunk_cache(ncid, varid, real_size, real_nelems, real_preemption); diff --git a/nc_test/tst_diskless.c b/nc_test/tst_diskless.c index 13ae21b354..54f41f8770 100644 --- a/nc_test/tst_diskless.c +++ b/nc_test/tst_diskless.c @@ -126,7 +126,7 @@ printf("*** testing diskless file with scalar vars..."); int ndims_in, nvars_in, natts_in, unlimdimid_in; char name_in[NC_MAX_NAME + 1]; nc_type type_in; - float float_data = 3.14, float_data_in; + float float_data = 3.14f, float_data_in; int int_data = 42, int_data_in; short short_data = 2, short_data_in; @@ -176,7 +176,7 @@ printf("*** testing diskless file with scalar vars..."); if(!usenetcdf4 && persist) { int ncid, varid0, varid1, varid2; - float float_data = 3.14, float_data_in; + float float_data = 3.14f, float_data_in; int int_data = 42, int_data_in; short short_data = 2, short_data_in; @@ -221,7 +221,7 @@ printf("*** testing diskless file with scalar vars..."); size_t start[1] = {0}; size_t count[1] = {DIM1_LEN}; int i; - float float_data = 42.22, float_data_in; + float float_data = 42.22f, float_data_in; /* This is some really important data that I want to save. */ for (i = 0; i < DIM1_LEN; i++) @@ -301,7 +301,7 @@ printf("*** testing diskless file with scalar vars..."); int ndims_in, nvars_in, natts_in, unlimdimid_in; char name_in[NC_MAX_NAME + 1]; nc_type type_in; - float float_data = 3.14, float_data_in; + float float_data = 3.14f, float_data_in; int int_data = 42, int_data_in; short short_data = 2, short_data_in; diff --git a/nc_test/util.c b/nc_test/util.c index 4399763754..7bb16e44f3 100644 --- a/nc_test/util.c +++ b/nc_test/util.c @@ -88,8 +88,8 @@ inRange_float(const double value, const nc_type xtype) case NC_UBYTE: min = 0; max = X_UCHAR_MAX; break; case NC_USHORT: min = 0; max = X_USHORT_MAX; break; case NC_UINT: min = 0; max = X_UINT_MAX; break; - case NC_INT64: min = X_INT64_MIN; max = X_INT64_MAX; break; - case NC_UINT64: min = 0; max = X_UINT64_MAX; break; + case NC_INT64: min = X_INT64_MIN; max = (double)X_INT64_MAX; break; + case NC_UINT64: min = 0; max = (double)X_UINT64_MAX; break; default: assert(0); } if(!( value >= min && value <= max)) { diff --git a/nc_test4/cdm_sea_soundings.c b/nc_test4/cdm_sea_soundings.c index f7da504ce9..b2e2f681cd 100644 --- a/nc_test4/cdm_sea_soundings.c +++ b/nc_test4/cdm_sea_soundings.c @@ -28,11 +28,11 @@ main(int argc, char **argv) /* Create phony data. */ for (i = 0; i < DIM_LEN; i++) { - if (!(data[i].temp_vl.p = malloc(sizeof(float) * (i + 1)))) + if (!(data[i].temp_vl.p = malloc(sizeof(float) * (size_t)(i + 1)))) return NC_ENOMEM; for (j = 0; j < i + 1; j++) - ((float *)(data[i].temp_vl.p))[j] = 23.5 - j; - data[i].temp_vl.len = i + 1; + ((float *)(data[i].temp_vl.p))[j] = 23.5f - (float)j; + data[i].temp_vl.len = (size_t)i + 1; } printf("\n*** Testing netcdf-4 CDM compliance: sea soundings.\n"); diff --git a/nc_test4/tst_atts3.c b/nc_test4/tst_atts3.c index 0c0bd621b2..84726014d4 100644 --- a/nc_test4/tst_atts3.c +++ b/nc_test4/tst_atts3.c @@ -1215,7 +1215,7 @@ create_file() check_err(stat,__LINE__,__FILE__); } { /* Gf */ - static const float Gf_att[5] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36, 531} ; + static const float Gf_att[5] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f, 531} ; stat = nc_put_att_float(ncid, NC_GLOBAL, "Gf", NC_FLOAT, 5, Gf_att); check_err(stat,__LINE__,__FILE__); } @@ -1247,7 +1247,7 @@ create_file() check_err(stat,__LINE__,__FILE__); } { /* f */ - static const float i_f_att[4] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36} ; + static const float i_f_att[4] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f} ; stat = nc_put_att_float(ncid, i_id, "f", NC_FLOAT, 4, i_f_att); check_err(stat,__LINE__,__FILE__); } @@ -1330,7 +1330,7 @@ create_file() } { - float fr_data[2] = {-3.4028231e+38, 3.4028231e+38} ; + float fr_data[2] = {-3.4028231e+38f, 3.4028231e+38f} ; size_t fr_startset[1] = {0} ; size_t fr_countset[1] = {2} ; stat = nc_put_vara(ncid, fr_id, fr_startset, fr_countset, fr_data); @@ -1378,7 +1378,7 @@ create_file() } { - float f1_data[1] = {-3.4028231e+38} ; + float f1_data[1] = {-3.4028231e+38f} ; size_t f1_startset[1] = {0} ; size_t f1_countset[1] = {1} ; stat = nc_put_vara(ncid, f1_id, f1_startset, f1_countset, f1_data); @@ -1426,7 +1426,7 @@ create_file() } { - float f2_data[2] = {-3.4028231e+38, 3.4028231e+38} ; + float f2_data[2] = {-3.4028231e+38f, 3.4028231e+38f} ; size_t f2_startset[1] = {0} ; size_t f2_countset[1] = {2} ; stat = nc_put_vara(ncid, f2_id, f2_startset, f2_countset, f2_data); @@ -1474,7 +1474,7 @@ create_file() } { - float f3_data[3] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36} ; + float f3_data[3] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f} ; size_t f3_startset[1] = {0} ; size_t f3_countset[1] = {3} ; stat = nc_put_vara(ncid, f3_id, f3_startset, f3_countset, f3_data); @@ -1522,7 +1522,7 @@ create_file() } { - float f4_data[4] = {-3.4028231e+38, 3.4028231e+38, -9.96921e+36, 9.96921e+36} ; + float f4_data[4] = {-3.4028231e+38f, 3.4028231e+38f, -9.96921e+36f, 9.96921e+36f} ; size_t f4_startset[1] = {0} ; size_t f4_countset[1] = {4} ; stat = nc_put_vara(ncid, f4_id, f4_startset, f4_countset, f4_data); diff --git a/nc_test4/tst_chunks2.c b/nc_test4/tst_chunks2.c index f166a500eb..0499d6f392 100644 --- a/nc_test4/tst_chunks2.c +++ b/nc_test4/tst_chunks2.c @@ -50,7 +50,7 @@ calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste) #endif /* Percent of the chunked file that is wasted space. */ - *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0; + *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0f; #ifdef PRINT_CHUNK_WASTE_REPORT printf("\ndimlen\tchunksize\tnum_chunks\n"); diff --git a/nc_test4/tst_compounds.c b/nc_test4/tst_compounds.c index f91c210021..731ed8bc4f 100644 --- a/nc_test4/tst_compounds.c +++ b/nc_test4/tst_compounds.c @@ -546,8 +546,8 @@ main(int argc, char **argv) hr_data_out[i].starfleet_id = i; hr_data_out[i].svc_rec = data[i]; if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; - hr_data_out[i].max_temp = 99.99; - hr_data_out[i].min_temp = -9.99; + hr_data_out[i].max_temp = 99.99f; + hr_data_out[i].min_temp = -9.99f; hr_data_out[i].percent_transporter_errosion = .030303; } @@ -659,10 +659,10 @@ main(int argc, char **argv) /* Initialize data. */ for (i = 0; i < DIM6_LEN; i++) { - obsdata[i].day = 15 * i + 1; - obsdata[i].elev = 2 * i + 1; + obsdata[i].day = 15 * (char)i + 1; + obsdata[i].elev = 2 * (short)i + 1; obsdata[i].count = 2 * i + 1; - obsdata[i].relhum = 2.0 * i + 1; + obsdata[i].relhum = 2.0f * (float)i + 1; obsdata[i].time = 2.0 * i + 1; } missing_val.day = 99; @@ -863,7 +863,7 @@ main(int argc, char **argv) for (i = 0; i < DIM1_LEN; i++) { if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; - hr_data_out[i].max_temp = 99.99; + hr_data_out[i].max_temp = 99.99f; } /* Create a file with a nested compound type attribute and variable. */ @@ -927,7 +927,7 @@ main(int argc, char **argv) /* Create some phony data. */ for (i = 0; i < DIM1_LEN; i++) { - hr_data_out[i].max_temp = 99.99; + hr_data_out[i].max_temp = 99.99f; for (j = 0; j < ARRAY_LEN; j++) hr_data_out[i].name[j] = j; } diff --git a/nc_test4/tst_compounds2.c b/nc_test4/tst_compounds2.c index 7b1af1de96..8a9fadb4db 100644 --- a/nc_test4/tst_compounds2.c +++ b/nc_test4/tst_compounds2.c @@ -49,7 +49,7 @@ main(int argc, char **argv) /* Create some phony data. */ for (i = 0; i < DIM1_LEN; i++) { - data_out[i].y = 99.99; + data_out[i].y = 99.99f; for (j = 0; j < ARRAY_LEN; j++) data_out[i].x[j] = j; } diff --git a/nc_test4/tst_compounds3.c b/nc_test4/tst_compounds3.c index c09cb0b448..d388658bbb 100644 --- a/nc_test4/tst_compounds3.c +++ b/nc_test4/tst_compounds3.c @@ -136,7 +136,7 @@ main(int argc, char **argv) for (i = 0; i < ATT_LEN; i++) { - a1_att[i].s1.x = 13.3; + a1_att[i].s1.x = 13.3f; a1_att[i].s1.y = 13.3; } /* Create a file with two groups, define a type in each group, diff --git a/nc_test4/tst_converts2.c b/nc_test4/tst_converts2.c index 0f3bd92258..da6006a163 100644 --- a/nc_test4/tst_converts2.c +++ b/nc_test4/tst_converts2.c @@ -215,7 +215,7 @@ main(int argc, char **argv) if (uint64_in != ivalue) ERR; if (nc_get_var_float(ncid, varid, &float_in)) ERR; - if(fabs( (float_in-X_INT_MAX) - ((float)ivalue - X_INT_MAX)) > 1) ERR; + if(fabs( (float_in-(float)X_INT_MAX) - ((float)ivalue - (float)X_INT_MAX)) > 1) ERR; if (nc_get_var_double(ncid, varid, &double_in)) ERR; if (double_in != (double)ivalue) ERR; diff --git a/nc_test4/tst_coords.c b/nc_test4/tst_coords.c index 000b8a632b..f7133a57f9 100644 --- a/nc_test4/tst_coords.c +++ b/nc_test4/tst_coords.c @@ -38,8 +38,8 @@ main(int argc, char **argv) #define LAT_NAME "lat" #define LON_NAME "lon" #define NVARS 2 -#define START_LAT 25.0 -#define START_LON -125.0 +#define START_LAT 25.0f +#define START_LON -125.0f { int ncid, lon_dimid, lat_dimid; int lat_varid, lon_varid; @@ -52,9 +52,9 @@ main(int argc, char **argv) /* Initialize coord data. */ for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5. * lat; + lats[lat] = START_LAT + 5.f * (float)lat; for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5. * lon; + lons[lon] = START_LON + 5.f * (float)lon; /* Create file with two dimensions. */ if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; @@ -103,7 +103,7 @@ main(int argc, char **argv) #define DIM0_NAME "d0" #define CACHE_SIZE 1000000 #define CACHE_NELEMS 1009 -#define CACHE_PREEMPTION .90 +#define CACHE_PREEMPTION .90f int ncid, dimid, varid; char name_in[NC_MAX_NAME + 1]; @@ -413,12 +413,12 @@ main(int argc, char **argv) check_err(stat,__LINE__,__FILE__); } { /* missing_value */ - static const float pr_missing_value_att[1] = {1e+20} ; + static const float pr_missing_value_att[1] = {1e+20f} ; stat = nc_put_att_float(root_grp, pr_id, "missing_value", NC_FLOAT, 1, pr_missing_value_att); check_err(stat,__LINE__,__FILE__); } { /* _FillValue */ - static const float pr_FillValue_att[1] = {1e+20} ; + static const float pr_FillValue_att[1] = {1e+20f} ; stat = nc_put_att_float(root_grp, pr_id, "_FillValue", NC_FLOAT, 1, pr_FillValue_att); check_err(stat,__LINE__,__FILE__); } diff --git a/nc_test4/tst_dims.c b/nc_test4/tst_dims.c index d821e20f73..f64ba66d59 100644 --- a/nc_test4/tst_dims.c +++ b/nc_test4/tst_dims.c @@ -849,9 +849,9 @@ main(int argc, char **argv) /* Lats and lons suitable for some South American data. */ for (lat[0] = 40.0, i = 1; i < LAT_LEN; i++) - lat[i] = lat[i - 1] + .5; + lat[i] = (float)lat[i - 1] + .5f; for (lon[0] = 20.0, i = 1; i < LON_LEN; i++) - lon[i] = lon[i - 1] + 1.5; + lon[i] = (float)lon[i - 1] + 1.5f; /* Some phoney 2D pressure data. */ for (i = 0; i < LAT_LEN; i++) @@ -1002,9 +1002,9 @@ main(int argc, char **argv) /* Some phony 1D lats and lons. */ for (i = 0; i < LAT_LEN; i++) - lat[i] = i * 5.; + lat[i] = (float)i * 5.f; for (i = 0; i < LON_LEN; i++) - lon[i] = i * 5.; + lon[i] = (float)i * 5.f; if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index e0f7ddf302..f2ad31bedd 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -209,7 +209,7 @@ main(int argc, char **argv) int dimids_var[2], var_type; size_t dim_len; char dim_name[NC_MAX_NAME+1], var_name[NC_MAX_NAME+1]; - float float_in, float_out = 99.99; + float float_in, float_out = 99.99f; int int_in, int_out = -9999; /* Create a file, this time with attributes. */ diff --git a/nc_test4/tst_put_vars.c b/nc_test4/tst_put_vars.c index 62f59ff80e..1ce011c2a3 100644 --- a/nc_test4/tst_put_vars.c +++ b/nc_test4/tst_put_vars.c @@ -42,7 +42,7 @@ main() int dimids[NDIMS]; for (i = 0; i < (X_LEN * Y_LEN * Z_LEN); i++) - mydata[i] = i; + mydata[i] = (float)i; /* create the file in NetCDF-4 format */ if ((retval = nc_create(FILE_NAME, NC_NETCDF4, &ncid))) diff --git a/nc_test4/tst_quantize.c b/nc_test4/tst_quantize.c index 47c8ce2eef..eb43c30437 100644 --- a/nc_test4/tst_quantize.c +++ b/nc_test4/tst_quantize.c @@ -277,7 +277,7 @@ main(int argc, char **argv) { int ncid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -394,7 +394,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -485,7 +485,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567}; + float float_data[DIM_LEN_5] = {1.11111111f, 1.0f, 9.99999999f, 12345.67f, .1234567f}; double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; int x; @@ -616,7 +616,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_1] = {1.1111111}; + float float_data[DIM_LEN_1] = {1.1111111f}; double double_data[DIM_LEN_1] = {1.111111111111}; printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); @@ -733,7 +733,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, 1.0, 9.99999999, 12345.67, .1234567}; + float float_data[DIM_LEN_5] = {1.11111111f, 1.0, 9.99999999f, 12345.67f, .1234567f}; double double_data[DIM_LEN_5] = {1.1111111, 1.0, 9.999999999, 1234567890.12345, 123456789012345.0}; int x; @@ -862,7 +862,7 @@ main(int argc, char **argv) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, NC_FILL_FLOAT, 9.99999999, 12345.67, NC_FILL_FLOAT}; + float float_data[DIM_LEN_5] = {1.11111111f, NC_FILL_FLOAT, 9.99999999f, 12345.67f, NC_FILL_FLOAT}; double double_data[DIM_LEN_5] = {1.1111111, NC_FILL_DOUBLE, 9.999999999, 1234567890.12345, NC_FILL_DOUBLE}; int x; @@ -987,13 +987,13 @@ main(int argc, char **argv) SUMMARIZE_ERR; printf("\t**** testing more quantization values with custom fill values...\n"); { -#define CUSTOM_FILL_FLOAT 99.99999 +#define CUSTOM_FILL_FLOAT 99.99999f #define CUSTOM_FILL_DOUBLE -99999.99999 for (q = 0; q < NUM_QUANTIZE_MODES; q++) { int ncid, dimid, varid1, varid2; int quantize_mode_in, nsd_in; - float float_data[DIM_LEN_5] = {1.11111111, CUSTOM_FILL_FLOAT, 9.99999999, 12345.67, CUSTOM_FILL_FLOAT}; + float float_data[DIM_LEN_5] = {1.11111111f, CUSTOM_FILL_FLOAT, 9.99999999f, 12345.67f, CUSTOM_FILL_FLOAT}; double double_data[DIM_LEN_5] = {1.1111111, CUSTOM_FILL_DOUBLE, 9.999999999, 1234567890.12345, CUSTOM_FILL_DOUBLE}; float custom_fill_float = CUSTOM_FILL_FLOAT; double custom_fill_double = CUSTOM_FILL_DOUBLE; @@ -1311,8 +1311,8 @@ main(int argc, char **argv) ERR; for (i = 0; i < DIM_LEN_SIMPLE; i++) { - float_data[i] = 1.5 * i; - double_data[i] = 1.5 * i; + float_data[i] = 1.5f * (float)i; + double_data[i] = 1.5 * (double)i; } /* Create the file. */ diff --git a/nc_test4/tst_unlim_vars.c b/nc_test4/tst_unlim_vars.c index e6f157a1ce..e488f36ad6 100644 --- a/nc_test4/tst_unlim_vars.c +++ b/nc_test4/tst_unlim_vars.c @@ -51,7 +51,7 @@ main(int argc, char **argv) for (time = 0; time < NUM_TIMESTEPS; time++) for (lat = 0; lat < LAT_LEN; lat++) for (lon = 0; lon < LON_LEN; lon++) - data_out[time][lat][lon] = 25.5 + lat + lon + time; + data_out[time][lat][lon] = 25.5f + (float)(lat + lon + time); /* Create a file with a 3D surface temp variable. */ if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; diff --git a/nc_test4/tst_vars.c b/nc_test4/tst_vars.c index a605da71ef..f94fc0dcbb 100644 --- a/nc_test4/tst_vars.c +++ b/nc_test4/tst_vars.c @@ -56,10 +56,10 @@ #define NVARS_EX 4 /* These are used to construct some example data. */ -#define SAMPLE_PRESSURE 900 -#define SAMPLE_TEMP 9.0 -#define START_LAT 25.0 -#define START_LON -125.0 +#define SAMPLE_PRESSURE 900.f +#define SAMPLE_TEMP 9.0f +#define START_LAT 25.0f +#define START_LON -125.0f /* For the units attributes. */ #define UNITS "units" @@ -97,16 +97,16 @@ create_4D_example(char *file_name, int cmode) * would have some real data to write, for example, model * output. */ for (lat = 0; lat < NLAT; lat++) - lats[lat] = START_LAT + 5.*lat; + lats[lat] = START_LAT + 5.f*(float)lat; for (lon = 0; lon < NLON; lon++) - lons[lon] = START_LON + 5.*lon; + lons[lon] = START_LON + 5.f*(float)lon; for (lvl = 0; lvl < NLVL; lvl++) for (lat = 0; lat < NLAT; lat++) for (lon = 0; lon < NLON; lon++) { - pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + i; - temp_out[lvl][lat][lon] = SAMPLE_TEMP + i++; + pres_out[lvl][lat][lon] = SAMPLE_PRESSURE + (float)i; + temp_out[lvl][lat][lon] = SAMPLE_TEMP + (float)i++; } /* Create the file. */ @@ -307,7 +307,7 @@ main(int argc, char **argv) unsigned short ushort_out[DIM1_LEN][DIM2_LEN] = {{110, 128, 255},{110, 128, 255}}; short short_in[DIM1_LEN][DIM2_LEN], short_out[DIM1_LEN][DIM2_LEN] = {{-110, -128, 255},{-110, -128, 255}}; int int_in[DIM1_LEN][DIM2_LEN], int_out[DIM1_LEN][DIM2_LEN] = {{0, 128, 255},{0, 128, 255}}; - float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1, 9999.99, 100.001},{-.1, 9999.99, 100.001}}; + float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1f, 9999.99f, 100.001f},{-.1f, 9999.99f, 100.001f}}; double double_in[DIM1_LEN][DIM2_LEN], double_out[DIM1_LEN][DIM2_LEN] = {{0.02, .1128, 1090.1},{0.02, .1128, 1090.1}}; unsigned int uint_in[DIM1_LEN][DIM2_LEN], uint_out[DIM1_LEN][DIM2_LEN] = {{0, 128, 255},{0, 128, 255}}; long long int64_in[DIM1_LEN][DIM2_LEN], int64_out[DIM1_LEN][DIM2_LEN] = {{-111, 777, 100},{-111, 777, 100}}; diff --git a/nc_test4/tst_xplatform2.c b/nc_test4/tst_xplatform2.c index 02b7dc7067..6d2043d9f4 100644 --- a/nc_test4/tst_xplatform2.c +++ b/nc_test4/tst_xplatform2.c @@ -312,7 +312,7 @@ main(int argc, char **argv) return NC_ENOMEM; for (j = 0; j < vlen_of_comp_out[i].len; j++) { - ((struct s1 *)vlen_of_comp_out[i].p)[j].x = 42.42; + ((struct s1 *)vlen_of_comp_out[i].p)[j].x = 42.42f; ((struct s1 *)vlen_of_comp_out[i].p)[j].y = 2.0; } } @@ -322,7 +322,7 @@ main(int argc, char **argv) for (i = 0; i < DIM2_LEN; i++) for (j = 0; j < NUM_S1; j++) { - comp_array_of_comp_out[i].data[j].x = 42.42; + comp_array_of_comp_out[i].data[j].x = 42.42f; comp_array_of_comp_out[i].data[j].y = 2.0; } @@ -336,7 +336,7 @@ main(int argc, char **argv) return NC_ENOMEM; for (k = 0; k < comp_array_of_vlen_of_comp_out[i].data[j].len; k++) { - ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].x = 42.42; + ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].x = 42.42f; ((struct s1 *)comp_array_of_vlen_of_comp_out[i].data[j].p)[k].y = 2.0; } } diff --git a/ncdap_test/t_dap3a.c b/ncdap_test/t_dap3a.c index bbfd36252e..4ededc15c8 100644 --- a/ncdap_test/t_dap3a.c +++ b/ncdap_test/t_dap3a.c @@ -317,7 +317,7 @@ static float int162float32_data[DIMSIZE]={0.000,256.000,512.000,768.000,1024.000 static int int32_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; static float int32tofloat32_data[DIMSIZE]={0.000,2048.000,4096.000,6144.000,8192.000,10240.000,12288.000,14336.000,16384.000,18432.000,20480.000,22528.000,24576.000,26624.000,28672.000,30720.000,32768.000,34816.000,36864.000,38912.000,40960.000,43008.000,45056.000,47104.000,49152.000}; static long int32toilong_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; -static float float32_data[DIMSIZE]={0.000,0.010,0.020,0.030,0.040,0.050,0.060,0.070,0.080,0.090,0.100,0.110,0.120,0.130,0.140,0.149,0.159,0.169,0.179,0.189,0.199,0.208,0.218,0.228,0.238}; +static float float32_data[DIMSIZE]={0.000f,0.010f,0.020f,0.030f,0.040f,0.050f,0.060f,0.070f,0.080f,0.090f,0.100f,0.110f,0.120f,0.130f,0.140f,0.149f,0.159f,0.169f,0.179f,0.189f,0.199f,0.208f,0.218f,0.228f,0.238f}; static double float64_data[DIMSIZE]={1.000,1.000,1.000,1.000,0.999,0.999,0.998,0.998,0.997,0.996,0.995,0.994,0.993,0.992,0.990,0.989,0.987,0.986,0.984,0.982,0.980,0.978,0.976,0.974,0.971}; #ifndef USE_NETCDF4 diff --git a/ncdap_test/test_cvt.c b/ncdap_test/test_cvt.c index 46f52de2a0..3485dd4f7a 100644 --- a/ncdap_test/test_cvt.c +++ b/ncdap_test/test_cvt.c @@ -316,7 +316,7 @@ static float int162float32_data[DIMSIZE]={0.000,256.000,512.000,768.000,1024.000 static int int32_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; static float int32tofloat32_data[DIMSIZE]={0.000,2048.000,4096.000,6144.000,8192.000,10240.000,12288.000,14336.000,16384.000,18432.000,20480.000,22528.000,24576.000,26624.000,28672.000,30720.000,32768.000,34816.000,36864.000,38912.000,40960.000,43008.000,45056.000,47104.000,49152.000}; static long int32toilong_data[DIMSIZE]={0,2048,4096,6144,8192,10240,12288,14336,16384,18432,20480,22528,24576,26624,28672,30720,32768,34816,36864,38912,40960,43008,45056,47104,49152}; -static float float32_data[DIMSIZE]={0.000,0.010,0.020,0.030,0.040,0.050,0.060,0.070,0.080,0.090,0.100,0.110,0.120,0.130,0.140,0.149,0.159,0.169,0.179,0.189,0.199,0.208,0.218,0.228,0.238}; +static float float32_data[DIMSIZE]={0.000f,0.010f,0.020f,0.030f,0.040f,0.050f,0.060f,0.070f,0.080f,0.090f,0.100f,0.110f,0.120f,0.130f,0.140f,0.149f,0.159f,0.169f,0.179f,0.189f,0.199f,0.208f,0.218f,0.228f,0.238f}; static double float64_data[DIMSIZE]={1.000,1.000,1.000,1.000,0.999,0.999,0.998,0.998,0.997,0.996,0.995,0.994,0.993,0.992,0.990,0.989,0.987,0.986,0.984,0.982,0.980,0.978,0.976,0.974,0.971}; #ifndef USE_NETCDF4 diff --git a/ncdap_test/test_varm3.c b/ncdap_test/test_varm3.c index 707bf53011..740d6fa6f6 100644 --- a/ncdap_test/test_varm3.c +++ b/ncdap_test/test_varm3.c @@ -33,33 +33,33 @@ TODO: Note that this test uses thredds server #define VAR "SST" static float expected_stride1[12] = { -29.430857, -29.403780, -29.325428, -29.578333, -29.660833, -29.378437, -29.151943, -29.109715, -29.114864, -29.550285, -29.542500, -29.500286 +29.430857f, +29.403780f, +29.325428f, +29.578333f, +29.660833f, +29.378437f, +29.151943f, +29.109715f, +29.114864f, +29.550285f, +29.542500f, +29.500286f }; static float expected_stride2[6] = { -29.430857, -29.325428, -29.660833, -29.151943, -29.114864, -29.542500 +29.430857f, +29.325428f, +29.660833f, +29.151943f, +29.114864f, +29.542500f }; static float expected_stride3[3] = { -29.430857, -29.378437, -29.542500 +29.430857f, +29.378437f, +29.542500f }; void diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 2ac4a73d9c..ae041547e4 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -7,6 +7,7 @@ *********************************************************************/ #include "config.h" /* for USE_NETCDF4 macro */ +#include #include #include #ifdef HAVE_GETOPT_H @@ -2310,7 +2311,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0) error("Suffix used for '-m' option value must be K, M, G, T, or P"); - option_copy_buffer_size = dval; + option_copy_buffer_size = (size_t)dval; break; } case 'h': /* non-default size of chunk cache */ @@ -2318,7 +2319,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0) error("Suffix used for '-h' option value must be K, M, G, T, or P"); - option_chunk_cache_size = dval; + option_chunk_cache_size = (size_t)dval; break; } case 'e': /* number of elements chunk cache can hold */ @@ -2326,7 +2327,7 @@ main(int argc, char**argv) double dval = double_with_suffix(optarg); /* "K" for kilobytes. "M" for megabytes, ... */ if(dval < 0 ) error("Suffix used for '-e' option value must be K, M, G, T, or P"); - option_chunk_cache_nelems = (long)dval; + option_chunk_cache_nelems = (size_t)dval; break; } case 'r': diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index f577079813..abaa20e387 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -699,7 +699,7 @@ pr_att_valsx( case NC_BYTE: case NC_SHORT: case NC_INT: - ii = vals[iel]; + ii = (int)vals[iel]; res = snprintf(gps, PRIM_LEN, "%d", ii); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); @@ -708,28 +708,28 @@ pr_att_valsx( case NC_UBYTE: case NC_USHORT: case NC_UINT: - ui = vals[iel]; + ui = (unsigned int)vals[iel]; res = snprintf(gps, PRIM_LEN, "%u", ui); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_INT64: - i64 = vals[iel]; + i64 = (int64_t)vals[iel]; res = snprintf(gps, PRIM_LEN, "%lld", i64); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_UINT64: - ui64 = vals[iel]; + ui64 = (uint64_t)vals[iel]; res = snprintf(gps, PRIM_LEN, "%llu", ui64); assert(res < PRIM_LEN); (void) strlcat(attvals, gps, attvalslen); (void) strlcat(attvals, iel < len-1 ? " " : "", attvalslen); break; case NC_FLOAT: - ff = vals[iel]; + ff = (float)vals[iel]; res = snprintf(gps, PRIM_LEN, float_attx_fmt, ff); assert(res < PRIM_LEN); tztrim(gps); /* trim trailing 0's after '.' */ @@ -847,7 +847,7 @@ pr_att( case NC_VLEN: /* because size returned for vlen is base type size, but we * need space to read array of vlen structs into ... */ - data = emalloc((att.len + 1) * sizeof(nc_vlen_t)); + data = emalloc((att.len + 1) * sizeof(nc_vlen_t)); break; case NC_OPAQUE: data = emalloc((att.len + 1) * type_size); diff --git a/ncdump/tst_create_files.c b/ncdump/tst_create_files.c index c8122042a9..e4cee78e40 100644 --- a/ncdump/tst_create_files.c +++ b/ncdump/tst_create_files.c @@ -226,8 +226,8 @@ main(int argc, char **argv) #define ATT_NAME_CMP "my_favorite_wind_speeds" #define COMPOUND_NAME "wind_vector" #define NUM_FAVS 3 -#define U_VALUE 13.3 -#define V_VALUE 12.2 +#define U_VALUE 13.3f +#define V_VALUE 12.2f { int ncid; diff --git a/ncdump/tst_vlen_data.c b/ncdump/tst_vlen_data.c index ad5d18388e..f11b3932b7 100644 --- a/ncdump/tst_vlen_data.c +++ b/ncdump/tst_vlen_data.c @@ -91,7 +91,7 @@ main(int argc, char **argv) array[i] = (float *) malloc(ncolumns * sizeof(float)); if(array[i] == NULL) ERR; for (j = 0; j < ncolumns; j++) { - array[i][j] = 10.0 * (i + 1) + j; + array[i][j] = 10.0f * (float)(i + 1) + (float)j; } } array[4][0] = missing_value; /* overwrite last row with missing for equality test */ diff --git a/nczarr_test/test_zchunks2.c b/nczarr_test/test_zchunks2.c index 7b471a2b9f..54963761c8 100644 --- a/nczarr_test/test_zchunks2.c +++ b/nczarr_test/test_zchunks2.c @@ -52,7 +52,7 @@ calculate_waste(int ndims, size_t *dimlen, size_t *chunksize, float *waste) #endif /* Percent of the chunked file that is wasted space. */ - *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0; + *waste = ((float)(chunked - unchunked) / (float)chunked) * 100.0f; #ifdef PRINT_CHUNK_WASTE_REPORT printf("\ndimlen\tchunksize\tnum_chunks\n"); From 0b9c9941bc4e14c6faabcfa437e9d36c93c96a9e Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 26 Oct 2023 16:29:43 +0100 Subject: [PATCH 49/77] CMake: Use helper libraries for nczarr tests Avoids rebuilding the same files multiple times --- nczarr_test/CMakeLists.txt | 68 +++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 26 deletions(-) diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index 09b013823b..ced017dbee 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -55,36 +55,54 @@ ENDIF() IF(ENABLE_TESTS) - SET(COMMONSRC ut_util.c ut_test.c ${XGETOPTSRC}) - - SET(TSTCOMMONSRC test_utils.c test_utils.h ${XGETOPTSRC}) - - INCLUDE_DIRECTORIES(../libnczarr ../plugins ${CMAKE_SOURCE_DIR}/nczarr_test) + add_library(ut_util ut_util.c ut_test.c ${XGETOPTSRC}) + target_include_directories(ut_util PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) + target_link_libraries(ut_util PUBLIC netcdf ${ALL_TLL_LIBS}) + + add_library(test_utils test_utils.c test_utils.h ${XGETOPTSRC}) + target_include_directories(test_utils PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) + target_link_libraries(test_utils PUBLIC netcdf ${ALL_TLL_LIBS}) + + set_target_properties(ut_util test_utils PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR} + RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR} + ) + + macro(build_bin_test_with_util_lib F UTIL_LIB) + build_bin_test(${F}) + target_link_libraries(${F} ${UTIL_LIB}) + endmacro() + + macro(add_bin_test_with_util_lib PREFIX F UTIL_LIB) + add_bin_test(${PREFIX} ${F}) + target_link_libraries(${PREFIX}_${F} ${UTIL_LIB}) + endmacro() # Base tests # The tests are set up as a combination of shell scripts and executables that # must be run in a particular order. It is painful but will use macros to help # keep it from being too bad. - BUILD_BIN_TEST(ut_map ${COMMONSRC}) - BUILD_BIN_TEST(ut_mapapi ${COMMONSRC}) - BUILD_BIN_TEST(ut_json ${COMMONSRC}) + build_bin_test_with_util_lib(ut_map ut_util) + build_bin_test_with_util_lib(ut_mapapi ut_util) + build_bin_test_with_util_lib(ut_json ut_util) - BUILD_BIN_TEST(test_fillonlyz ${TSTCOMMONSRC}) - BUILD_BIN_TEST(test_quantize ${TSTCOMMONSRC}) - BUILD_BIN_TEST(test_notzarr ${TSTCOMMONSRC}) + build_bin_test_with_util_lib(test_fillonlyz test_utils) + build_bin_test_with_util_lib(test_quantize test_utils) + build_bin_test_with_util_lib(test_notzarr test_utils) # ADD_BIN_TEST(nczarr_test test_endians ${TSTCOMMONSRC}) # Unlimited Tests IF(USE_HDF5) - ADD_BIN_TEST(nczarr_test test_unlim_vars ${TSTCOMMONSRC}) - ADD_BIN_TEST(nczarr_test test_put_vars_two_unlim_dim ${TSTCOMMONSRC}) - BUILD_BIN_TEST(test_zchunks ${COMMONSRC}) - BUILD_BIN_TEST(test_zchunks2 ${COMMONSRC}) - BUILD_BIN_TEST(test_zchunks3 ${COMMONSRC}) - BUILD_BIN_TEST(test_unlim_io ${TSTCOMMONSRC}) - BUILD_BIN_TEST(test_chunking ${TSTCOMMONSRC}) + add_bin_test_with_util_lib(nczarr_test test_unlim_vars test_utils) + add_bin_test_with_util_lib(nczarr_test test_put_vars_two_unlim_dim test_utils) + build_bin_test_with_util_lib(test_zchunks ut_util) + build_bin_test_with_util_lib(test_zchunks2 ut_util) + build_bin_test_with_util_lib(test_zchunks3 ut_util) + build_bin_test_with_util_lib(test_unlim_io test_utils) + build_bin_test_with_util_lib(test_chunking test_utils) ADD_SH_TEST(nczarr_test run_nccopyz) ADD_SH_TEST(nczarr_test run_unlim_io) ADD_SH_TEST(nczarr_test run_nccopy5) @@ -99,11 +117,11 @@ IF(ENABLE_TESTS) # Helper programs for testing BUILD_BIN_TEST(zhex) - BUILD_BIN_TEST(zisjson ${COMMONSRC}) + build_bin_test_with_util_lib(zisjson ut_util) TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr) - BUILD_BIN_TEST(zs3parse ${COMMONSRC}) + build_bin_test_with_util_lib(zs3parse ut_util) TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr) - BUILD_BIN_TEST(zmapio ${COMMONSRC}) + build_bin_test_with_util_lib(zmapio ut_util) IF(ENABLE_S3 AND NOT WITH_S3_TESTING STREQUAL "NO") # Helper programs for testing @@ -132,9 +150,7 @@ IF(ENABLE_TESTS) endif() if(ENABLE_NCDUMPCHUNKS) - SET(ncdumpchunks_SOURCE ncdumpchunks.c) - SET(ncdumpchunks_SOURCE ${ncdumpchunks_SOURCE} ${XGETOPTSRC}) - BUILD_BIN_TEST(ncdumpchunks ${ncdumpchunks_SOURCE}) + build_bin_test_with_util_lib(ncdumpchunks ut_util) endif() IF(BUILD_UTILITIES) @@ -145,7 +161,7 @@ IF(ENABLE_TESTS) if(LARGE_FILE_TESTS) BUILD_BIN_TEST(test_readcaching}) BUILD_BIN_TEST(test_writecaching}) - BUILD_BIN_TEST(test_chunkcases ${TSTCOMMONSRC}) + build_bin_test_with_util_lib(test_chunkcases test_utils) add_sh_test(nczarr_test run_cachetest) add_sh_test(nczarr_test run_chunkcases) ENDIF() @@ -175,7 +191,7 @@ IF(ENABLE_TESTS) build_bin_test(test_nczfilter) build_bin_test(test_filter_vlen) build_bin_test(testfilter) - build_bin_test(testfilter_misc) + build_bin_test_with_util_lib(testfilter_misc test_utils) build_bin_test(testfilter_multi) build_bin_test(testfilter_order) build_bin_test(testfilter_repeat) From 8d422704ec0efcacf54b2685916ab405a42a7d9b Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Fri, 27 Oct 2023 12:20:46 -0600 Subject: [PATCH 50/77] Replace exec_program with execute_process --- CMakeInstallation.cmake | 2 +- CMakeLists.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index 7463420b81..5967889c95 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -74,7 +74,7 @@ FIND_PROGRAM(NC_DPKG NAMES dpkg) IF(NC_DPKG) # Define a macro for getting the dpkg architecture. MACRO(getdpkg_arch arch) - exec_program("${NC_DPKG}" ARGS "--print-architecture" OUTPUT_VARIABLE "${arch}") + execute_process(COMMAND "${NC_DPKG}" "--print-architecture" OUTPUT_VARIABLE "${arch}" OUTPUT_STRIP_TRAILING_WHITESPACE) ENDMACRO(getdpkg_arch) getdpkg_arch(dpkg_arch) diff --git a/CMakeLists.txt b/CMakeLists.txt index ddb5ae4061..189f6d87c2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,7 +57,7 @@ SET(NC_DISPATCH_VERSION 5) find_program(UNAME NAMES uname) IF(UNAME) macro(getuname name flag) - exec_program("${UNAME}" ARGS "${flag}" OUTPUT_VARIABLE "${name}") + execute_process(COMMAND "${UNAME}" "${flag}" OUTPUT_VARIABLE "${name}" OUTPUT_STRIP_TRAILING_WHITESPACE) endmacro(getuname) getuname(osname -s) getuname(osrel -r) @@ -1411,7 +1411,7 @@ IF(ENABLE_TESTS) SET(HOSTNAME_ARG "-s") ENDIF() IF(HOSTNAME_CMD) - EXEC_PROGRAM(${HOSTNAME_CMD} ARGS "${HOSTNAME_ARG}" OUTPUT_VARIABLE HOSTNAME) + EXECUTE_PROCESS(COMMAND ${HOSTNAME_CMD} "${HOSTNAME_ARG}" OUTPUT_VARIABLE HOSTNAME OUTPUT_STRIP_TRAILING_WHITESPACE) SET(NC_CTEST_SITE "${HOSTNAME}" CACHE STRING "Hostname of test machine.") ENDIF() From 70ea45ad41b220cc3ae146158628d52fbb2fd229 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Mon, 30 Oct 2023 12:26:16 -0600 Subject: [PATCH 51/77] Remove stray character in cmake lfs tests for nczarr. --- nczarr_test/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index c38b86fe81..e58add623a 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -143,8 +143,8 @@ IF(ENABLE_TESTS) add_sh_test(nczarr_test run_ut_misc) add_sh_test(nczarr_test run_ncgen4) if(LARGE_FILE_TESTS) - BUILD_BIN_TEST(test_readcaching}) - BUILD_BIN_TEST(test_writecaching}) + BUILD_BIN_TEST(test_readcaching) + BUILD_BIN_TEST(test_writecaching) BUILD_BIN_TEST(test_chunkcases ${TSTCOMMONSRC}) add_sh_test(nczarr_test run_cachetest) add_sh_test(nczarr_test run_chunkcases) From b9adb697acd0430500d200978ffb23c23032bad0 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Wed, 1 Nov 2023 11:35:52 -0600 Subject: [PATCH 52/77] Working to get the proposed change working with Visual studio --- nczarr_test/CMakeLists.txt | 14 ++++++++++---- nczarr_test/ut_test.h | 16 ++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index a09063356a..1bdc2ab745 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -9,6 +9,7 @@ SET(abs_top_srcdir ${CMAKE_CURRENT_SOURCE_DIR}) remove_definitions(-DDLL_EXPORT) + # Create nczarr tests from nc_test4 tests macro(NCZARR_SH_TEST basename src) FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../${src}/tst_${basename}.sh SHSOURCE) @@ -55,11 +56,13 @@ ENDIF() IF(ENABLE_TESTS) - add_library(ut_util ut_util.c ut_test.c ${XGETOPTSRC}) + add_library(ut_util STATIC ut_util.c ut_test.c ut_includes.h ut_test.h ${XGETOPTSRC}) + target_compile_definitions(ut_util PUBLIC -DDLL_EXPORT -DDLL_NETCDF) target_include_directories(ut_util PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(ut_util PUBLIC netcdf ${ALL_TLL_LIBS}) - add_library(test_utils test_utils.c test_utils.h ${XGETOPTSRC}) + add_library(test_utils STATIC test_utils.c test_utils.h ${XGETOPTSRC}) + target_compile_definitions(test_utils PUBLIC -DDLL_EXPORT -DDLL_NETCDF) target_include_directories(test_utils PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(test_utils PUBLIC netcdf ${ALL_TLL_LIBS}) @@ -71,12 +74,15 @@ IF(ENABLE_TESTS) macro(build_bin_test_with_util_lib F UTIL_LIB) build_bin_test(${F}) - target_link_libraries(${F} ${UTIL_LIB}) + target_compile_definitions(${F} PUBLIC -DDLL_NETCDF) + target_link_libraries(${F} ${UTIL_LIB} ${ALL_TLL_LIBS}) endmacro() macro(add_bin_test_with_util_lib PREFIX F UTIL_LIB) add_bin_test(${PREFIX} ${F}) - target_link_libraries(${PREFIX}_${F} ${UTIL_LIB}) + target_compile_definitions(${PREFIX}_${F} PUBLIC -DDLL_NETCDF) + target_link_libraries(${PREFIX}_${F} ${UTIL_LIB} ${ALL_TLL_LIBS}) + endmacro() # Base tests diff --git a/nczarr_test/ut_test.h b/nczarr_test/ut_test.h index 63ecad3cba..efe26c3658 100644 --- a/nczarr_test/ut_test.h +++ b/nczarr_test/ut_test.h @@ -62,18 +62,18 @@ struct Test { int (*test)(void); }; -extern struct UTOptions utoptions; +EXTERNL struct UTOptions utoptions; #define NCCHECK(expr) nccheck((expr),__LINE__) -extern void usage(int err); -extern int ut_init(int argc, char** argv, struct UTOptions* test); -extern void ut_final(void); +EXTERNL void usage(int err); +EXTERNL int ut_init(int argc, char** argv, struct UTOptions* test); +EXTERNL void ut_final(void); -extern void nccheck(int stat, int line); -extern char* makeurl(const char* file, NCZM_IMPL, struct UTOptions*); +EXTERNL void nccheck(int stat, int line); +EXTERNL char* makeurl(const char* file, NCZM_IMPL, struct UTOptions*); //extern int setup(int argc, char** argv); -extern struct Test* findtest(const char* cmd, struct Test* tests); -extern int runtests(const char** cmds, struct Test* tests); +EXTERNL struct Test* findtest(const char* cmd, struct Test* tests); +EXTERNL int runtests(const char** cmds, struct Test* tests); #endif /*ZTEST_H*/ From e92404f11f272b2dd1edb0193ed8dee3244972cf Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Wed, 1 Nov 2023 11:45:31 -0600 Subject: [PATCH 53/77] Add fenceposting so DLL flags are only introduced wwhen we are compiling DLL-based shared libraries. --- nczarr_test/CMakeLists.txt | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index 1bdc2ab745..6483f46444 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -57,12 +57,16 @@ ENDIF() IF(ENABLE_TESTS) add_library(ut_util STATIC ut_util.c ut_test.c ut_includes.h ut_test.h ${XGETOPTSRC}) - target_compile_definitions(ut_util PUBLIC -DDLL_EXPORT -DDLL_NETCDF) + IF(ENABLE_DLL) + target_compile_definitions(ut_util PUBLIC -DDLL_EXPORT -DDLL_NETCDF) + ENDIF(ENABLE_DLL) target_include_directories(ut_util PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(ut_util PUBLIC netcdf ${ALL_TLL_LIBS}) add_library(test_utils STATIC test_utils.c test_utils.h ${XGETOPTSRC}) - target_compile_definitions(test_utils PUBLIC -DDLL_EXPORT -DDLL_NETCDF) + IF(ENABLE_DLL) + target_compile_definitions(test_utils PUBLIC -DDLL_EXPORT -DDLL_NETCDF) + ENDIF(ENABLE_DLL) target_include_directories(test_utils PUBLIC ../libnczarr ../plugins ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(test_utils PUBLIC netcdf ${ALL_TLL_LIBS}) @@ -74,13 +78,17 @@ IF(ENABLE_TESTS) macro(build_bin_test_with_util_lib F UTIL_LIB) build_bin_test(${F}) - target_compile_definitions(${F} PUBLIC -DDLL_NETCDF) + IF(ENABLE_DLL) + target_compile_definitions(${F} PUBLIC -DDLL_NETCDF) + ENDIF(ENABLE_DLL) target_link_libraries(${F} ${UTIL_LIB} ${ALL_TLL_LIBS}) endmacro() macro(add_bin_test_with_util_lib PREFIX F UTIL_LIB) add_bin_test(${PREFIX} ${F}) - target_compile_definitions(${PREFIX}_${F} PUBLIC -DDLL_NETCDF) + IF(ENABLE_DLL) + target_compile_definitions(${PREFIX}_${F} PUBLIC -DDLL_NETCDF) + ENDIF(ENABLE_DLL) target_link_libraries(${PREFIX}_${F} ${UTIL_LIB} ${ALL_TLL_LIBS}) endmacro() From 1162f64d2a71c73527dcae84f744598e17cd86d5 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Mon, 1 May 2023 13:25:53 -0400 Subject: [PATCH 54/77] Fixed various UBSan warnings about invalid bit shifting Just made sure to use unsigned, so that a bit does not get shifted into a sign bit. --- libdispatch/ncexhash.c | 4 ++-- libdispatch/nclistmgr.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libdispatch/ncexhash.c b/libdispatch/ncexhash.c index ea3957a054..26375e8d8f 100644 --- a/libdispatch/ncexhash.c +++ b/libdispatch/ncexhash.c @@ -111,7 +111,7 @@ ncexinit(void) int i; bitmasks[0] = 0; for(i=1;ileaflen, leafavg); fprintf(stderr," load=%g",leafload); fprintf(stderr,"]\n"); - dirsize = (1<<(map->depth))*((unsigned long long)sizeof(void*)); + dirsize = (1ULL<<(map->depth))*((unsigned long long)sizeof(void*)); leafsize = (nleaves)*((unsigned long long)sizeof(NCexleaf)); total = dirsize + leafsize; fprintf(stderr,"\tsizeof(directory)=%llu sizeof(leaves)=%lld total=%lld\n", diff --git a/libdispatch/nclistmgr.c b/libdispatch/nclistmgr.c index 48b4eeb584..4069e87698 100644 --- a/libdispatch/nclistmgr.c +++ b/libdispatch/nclistmgr.c @@ -81,8 +81,8 @@ free_NCList(void) int add_to_NCList(NC* ncp) { - int i; - int new_id; + unsigned int i; + unsigned int new_id; if(nc_filelist == NULL) { if (!(nc_filelist = calloc(1, sizeof(NC*)*NCFILELISTLENGTH))) return NC_ENOMEM; @@ -96,7 +96,7 @@ add_to_NCList(NC* ncp) if(new_id == 0) return NC_ENOMEM; /* no more slots */ nc_filelist[new_id] = ncp; numfiles++; - ncp->ext_ncid = (new_id << ID_SHIFT); + ncp->ext_ncid = (int)(new_id << ID_SHIFT); return NC_NOERR; } From 64111c7f5ec9b6d4bae27bfab29e47dc457f02e1 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 4 Nov 2023 16:17:09 -0600 Subject: [PATCH 55/77] remove conflicts --- libdap4/d4data.c | 3 +++ libdap4/d4file.c | 55 ++++++------------------------------------ libdap4/d4parser.c | 3 ++- libdap4/d4varx.c | 7 +----- libdap4/ncd4dispatch.c | 4 +-- 5 files changed, 15 insertions(+), 57 deletions(-) diff --git a/libdap4/d4data.c b/libdap4/d4data.c index 234f3e3776..eae5f32ed8 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -78,6 +78,8 @@ NCD4_parcelvars(NCD4meta* meta, NCD4response* resp) var->data.response = resp; /* cross link */ } done: + nclistfree(toplevel); + nullfree(offset); return THROW(ret); } @@ -421,6 +423,7 @@ NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp) } } } + nclistfree(toplevel); resp->attrchecksumming = (attrfound ? 1 : 0); /* Infer checksums */ resp->inferredchecksumming = ((resp->attrchecksumming || resp->querychecksumming) ? 1 : 0); diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 471afa15d2..45ee9a4069 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -546,6 +546,7 @@ NCD4_newInfo(NCD4INFO** d4infop) if((info = calloc(1,sizeof(NCD4INFO)))==NULL) {ret = NC_ENOMEM; goto done;} info->platform.hostlittleendian = NCD4_isLittleEndian(); + info->responses = nclistnew(); if(d4infop) {*d4infop = info; info = NULL;} done: if(info) NCD4_reclaimInfo(info); @@ -570,13 +571,12 @@ NCD4_reclaimInfo(NCD4INFO* d4info) nclistfree(d4info->blobs); /* Reclaim dmr node tree */ NCD4_reclaimMeta(d4info->dmrmetadata); + /* Reclaim all responses */ for(i=0;iresponses);i++) { NCD4response* resp = nclistget(d4info->responses,i); NCD4_reclaimResponse(resp); } nclistfree(d4info->responses); - /* Reclaim all responses */ - NCD4_resetMeta(d4info->dmrmetadata); free(d4info); } @@ -597,10 +597,9 @@ NCD4_resetInfoForRead(NCD4INFO* d4info) */ if(d4info->substrate.filename != NULL) { unlink(d4info->substrate.filename); - } + } } - NCD4_resetMeta(d4info->dmrmetadata); - nullfree(d4info->dmrmetadata); + NCD4_reclaimMeta(d4info->dmrmetadata); d4info->dmrmetadata = NULL; } @@ -641,50 +640,8 @@ NCD4_reclaimResponse(NCD4response* d4resp) d4resp->controller = NULL; /* break link */ nullfree(d4resp->raw.memory); - - nullfree(serial->errdata); nullfree(serial->dmr); - nullfree(serial->dap); - /* clear all fields */ - memset(serial,0,sizeof(struct NCD4serial)); - - nullfree(d4resp->error.parseerror); - nullfree(d4resp->error.message); - nullfree(d4resp->error.context); - nullfree(d4resp->error.otherinfo); - memset(&d4resp->error,0,sizeof(d4resp->error)); - - free(d4resp); -} - - -/* Create an empty NCD4meta object for - use in subsequent calls - (is the the right src file to hold this?) -*/ - -int -NCD4_newMeta(NCD4INFO* info, NCD4meta** metap) -{ - int ret = NC_NOERR; - NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); - if(meta == NULL) return NC_ENOMEM; - meta->allnodes = nclistnew(); -#ifdef D4DEBUG - meta->debuglevel = 1; -#endif - meta->controller = info; - meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ - if(metap) {*metap = meta; meta = NULL;} - return THROW(ret); -} - -void -NCD4_reclaimMeta(NCD4meta* dataset) -{ - int i; - if(dataset == NULL) return; - NCD4_resetMeta(dataset); + nullfree(serial->errdata); for(i=0;iallnodes);i++) { NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); @@ -696,6 +653,7 @@ NCD4_reclaimMeta(NCD4meta* dataset) free(dataset); } +#if 0 void NCD4_resetMeta(NCD4meta* dataset) { @@ -708,3 +666,4 @@ NCD4_resetMeta(NCD4meta* dataset) nclistfree(dataset->blobs); #endif } +#endif diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index 776163a013..2948f80672 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -228,7 +228,8 @@ traverse(NCD4parser* parser, ncxml_t dom) if((ret=makeNode(parser,NULL,NULL,NCD4_GROUP,NC_NULL,&parser->metadata->root))) goto done; parser->metadata->root->group.isdataset = 1; parser->metadata->root->meta.id = parser->metadata->ncid; - parser->metadata->groupbyid = nclistnew(); + if(parser->metadata->groupbyid == NULL) + parser->metadata->groupbyid = nclistnew(); SETNAME(parser->metadata->root,"/"); xattr = ncxml_attr(dom,"name"); if(xattr != NULL) parser->metadata->root->group.datasetname = xattr; diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index 1701718004..8956486e7d 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -286,15 +286,9 @@ static int mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming) { int i, ret = NC_NOERR; - NCD4node* dmrroot = dmrmeta->root; - NClist* dmrtop = NULL; /* top variables in dmr tree */ NCD4node* daproot = dapmeta->root; NClist* daptop = NULL; /* top variables in dap tree */ - /* Get top level variables from the dmr node tree */ - dmrtop = nclistnew(); - NCD4_getToplevelVars(dmrmeta,dmrroot,dmrtop); - /* Get top level variables from the dap node tree */ daptop = nclistnew(); NCD4_getToplevelVars(dapmeta,daproot,daptop); @@ -311,5 +305,6 @@ mapvars(NCD4meta* dapmeta, NCD4meta* dmrmeta, int inferredchecksumming) } done: + nclistfree(daptop); return THROW(ret); } diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 5f5ebbfec4..e7eefb40a4 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -388,11 +388,11 @@ NCD4_inq_attname(int ncid, int varid, int attnum, char* name) const NC_reservedatt* rsvp = NULL; if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret); + substrateid = makenc4id(ncp,ncid); + ret = nc_inq_attname(substrateid, varid, attnum, name); /* Is this a reserved attribute name? */ if(name && (rsvp = NCD4_lookupreserved(name))) return NC_EATTMETA; - substrateid = makenc4id(ncp,ncid); - ret = nc_inq_attname(substrateid, varid, attnum, name); return (ret); } From f8cb89e679566b375797f511a595a9b2ee4f2962 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 4 Nov 2023 16:42:36 -0600 Subject: [PATCH 56/77] ckp --- libdap4/d4file.c | 7 ++++--- libdap4/ncd4types.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 45ee9a4069..64e1abc6e2 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -154,7 +154,6 @@ NCD4_open(const char * path, int mode, NCD4_resetInfoForRead(d4info); /* Rebuild metadata */ if((ret = NCD4_newMeta(d4info,&d4info->dmrmetadata))) goto done; - meta = d4info->dmrmetadata; /* Capture response */ if((dmrresp = (NCD4response*)calloc(1,sizeof(NCD4response)))==NULL) @@ -188,10 +187,11 @@ NCD4_open(const char * path, int mode, } #endif - if((ret = NCD4_parse(meta,dmrresp,0))) goto done; + if((ret = NCD4_parse(d4info->dmrmetadata,dmrresp,0))) goto done; #ifdef D4DEBUGMETA { + meta = d4info->dmrmetadata; fprintf(stderr,"\n/////////////\n"); NCbytes* buf = ncbytesnew(); NCD4_print(meta,buf); @@ -204,7 +204,7 @@ NCD4_open(const char * path, int mode, #endif /* Build the substrate metadata */ - ret = NCD4_metabuild(meta,meta->ncid); + ret = NCD4_metabuild(d4info->dmrmetadata,meta->ncid); if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done; /* Remember the response */ @@ -634,6 +634,7 @@ NCD4_newResponse(NCD4INFO* info, NCD4response** respp) void NCD4_reclaimResponse(NCD4response* d4resp) { + int i; struct NCD4serial* serial = NULL; if(d4resp == NULL) return; serial = &d4resp->serial; diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index ca0a5118ac..7bed38a845 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -342,7 +342,7 @@ struct NCD4INFO { NCURI* dmruri; /* parse of rawuritext */ NCD4curl* curl; int inmemory; /* store fetched data in memory? */ - NCD4meta* dmrmetadata; + NCD4meta* dmrmetadata; /* Independent of responses */ NClist* responses; /* NClist all responses from this curl handle */ struct { /* Properties that are per-platform */ int hostlittleendian; /* 1 if the host is little endian */ From f878acf99eb7f645ea39c6d8ce1e82b0af2186f5 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 4 Nov 2023 20:23:55 -0600 Subject: [PATCH 57/77] fix memory leak --- libdap4/d4file.c | 43 ++++++++++++++++++++++++++++++++++++++++--- libdap4/d4varx.c | 43 +++++++++++++++++++++++++------------------ 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/libdap4/d4file.c b/libdap4/d4file.c index 64e1abc6e2..46c1a0fdbb 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -46,7 +46,6 @@ NCD4_open(const char * path, int mode, NCD4INFO* d4info = NULL; const char* value; NC* nc; - NCD4meta* meta = NULL; size_t len = 0; void* contents = NULL; NCD4response* dmrresp = NULL; @@ -204,7 +203,7 @@ NCD4_open(const char * path, int mode, #endif /* Build the substrate metadata */ - ret = NCD4_metabuild(d4info->dmrmetadata,meta->ncid); + ret = NCD4_metabuild(d4info->dmrmetadata,d4info->dmrmetadata->ncid); if(ret != NC_NOERR && ret != NC_EVARSIZE) goto done; /* Remember the response */ @@ -634,7 +633,6 @@ NCD4_newResponse(NCD4INFO* info, NCD4response** respp) void NCD4_reclaimResponse(NCD4response* d4resp) { - int i; struct NCD4serial* serial = NULL; if(d4resp == NULL) return; serial = &d4resp->serial; @@ -644,6 +642,45 @@ NCD4_reclaimResponse(NCD4response* d4resp) nullfree(serial->dmr); nullfree(serial->errdata); + /* clear all fields */ + memset(serial,0,sizeof(struct NCD4serial)); + + nullfree(d4resp->error.parseerror); + nullfree(d4resp->error.message); + nullfree(d4resp->error.context); + nullfree(d4resp->error.otherinfo); + memset(&d4resp->error,0,sizeof(d4resp->error)); + + free(d4resp); +} + +/* Create an empty NCD4meta object for + use in subsequent calls + (is the the right src file to hold this?) +*/ + +int +NCD4_newMeta(NCD4INFO* info, NCD4meta** metap) +{ + int ret = NC_NOERR; + NCD4meta* meta = (NCD4meta*)calloc(1,sizeof(NCD4meta)); + if(meta == NULL) return NC_ENOMEM; + meta->allnodes = nclistnew(); +#ifdef D4DEBUG + meta->debuglevel = 1; +#endif + meta->controller = info; + meta->ncid = info->substrate.nc4id; /* Transfer netcdf ncid */ + if(metap) {*metap = meta; meta = NULL;} + return THROW(ret); +} + +void +NCD4_reclaimMeta(NCD4meta* dataset) +{ + int i; + if(dataset == NULL) return; + for(i=0;iallnodes);i++) { NCD4node* node = (NCD4node*)nclistget(dataset->allnodes,i); reclaimNode(node); diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index 8956486e7d..b098855f83 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -187,24 +187,31 @@ getvarx(int gid, int varid, NCD4INFO** infop, NCD4node** varp, } /* Read and process the data */ - /* Setup the meta-data for the DAP */ - if((ret=NCD4_newMeta(info,&dapmeta))) goto done; - if((ret=NCD4_newResponse(info,&dapresp))) goto done; - dapresp->mode = NCD4_DAP; - nclistpush(info->responses,dapresp); - if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done; - /* Extract DMR and dechunk the data part */ - if((ret=NCD4_dechunk(dapresp))) goto done; - /* Process the dmr part */ - if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; - /* See if we are checksumming */ - if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; - /* connect variables and corresponding dap data */ - if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; - /* Process checksums and byte-order swapping */ - if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; - /* Transfer and process the data */ - if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; + + /* Setup the meta-data for the DAP */ + if((ret=NCD4_newMeta(info,&dapmeta))) goto done; + if((ret=NCD4_newResponse(info,&dapresp))) goto done; + dapresp->mode = NCD4_DAP; + nclistpush(info->responses,dapresp); + if((ret=NCD4_readDAP(info, info->controls.flags.flags, ceuri, dapresp))) goto done; + + /* Extract DMR and dechunk the data part */ + if((ret=NCD4_dechunk(dapresp))) goto done; + + /* Process the dmr part */ + if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; + + /* See if we are checksumming */ + if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; + + /* connect variables and corresponding dap data */ + if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; + + /* Process checksums and byte-order swapping */ + if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; + + /* Transfer and process the data */ + if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; validated: /* Return relevant info */ From adea80f3763ef59c1e2471dd3d5e045758d6e6b8 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 4 Nov 2023 21:32:33 -0600 Subject: [PATCH 58/77] Remove the execinfo capability re: Issue https://github.com/Unidata/netcdf-c/issues/2766 This attempt to programmatically dump stack never worked, so it is time to kill it off. --- RELEASE_NOTES.md | 1 + config.h.cmake.in | 3 --- configure.ac | 3 --- libdispatch/nclog.c | 32 -------------------------------- libhdf5/hdf5debug.c | 28 ---------------------------- 5 files changed, 1 insertion(+), 66 deletions(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 8d5eca680a..ee0d03cc94 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764). * Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). diff --git a/config.h.cmake.in b/config.h.cmake.in index 04d876ff17..233c2f1e27 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -462,9 +462,6 @@ with zip */ /* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */ #cmakedefine HDF5_UTF8_PATHS 1 -/* if true, backtrace support will be used. */ -#cmakedefine HAVE_EXECINFO_H 1 - /* if true, include JNA bug fix */ #cmakedefine JNA 1 diff --git a/configure.ac b/configure.ac index 19386fe141..b9c12a1ff0 100644 --- a/configure.ac +++ b/configure.ac @@ -1305,9 +1305,6 @@ AC_CHECK_HEADERS([sys/resource.h]) # See if we have ftw.h to walk directory trees AC_CHECK_HEADERS([ftw.h]) -# See if we can do stack tracing programmatically -AC_CHECK_HEADERS([execinfo.h]) - # Check for these functions... AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \ strdup strtoll strtoull \ diff --git a/libdispatch/nclog.c b/libdispatch/nclog.c index 5f6b8be7d8..c9664a23cf 100644 --- a/libdispatch/nclog.c +++ b/libdispatch/nclog.c @@ -21,10 +21,6 @@ #include #endif -#ifdef HAVE_EXECINFO_H -#include -#endif - #include "netcdf.h" #include "nclog.h" @@ -278,10 +274,6 @@ ncuntrace(const char* fcn, int err, const char* fmt, ...) vfprintf(nclog_global.nclogstream, fmt, args); fprintf(nclog_global.nclogstream, "\n" ); fflush(nclog_global.nclogstream); -#ifdef HAVE_EXECINFO_H - if(err != 0) - ncbacktrace(); -#endif } done: va_end(args); @@ -304,28 +296,4 @@ ncbreakpoint(int err) return err; } -#ifdef HAVE_EXECINFO_H -#define MAXSTACKDEPTH 100 -void -ncbacktrace(void) -{ - int j, nptrs; - void* buffer[MAXSTACKDEPTH]; - char **strings; - - if(getenv("NCBACKTRACE") == NULL) return; - nptrs = backtrace(buffer, MAXSTACKDEPTH); - strings = backtrace_symbols(buffer, nptrs); - if (strings == NULL) { - perror("backtrace_symbols"); - errno = 0; - return; - } - fprintf(stderr,"Backtrace:\n"); - for(j = 0; j < nptrs; j++) - fprintf(stderr,"%s\n", strings[j]); - free(strings); -} -#endif - /**@}*/ diff --git a/libhdf5/hdf5debug.c b/libhdf5/hdf5debug.c index 40871c148a..b4dace8d59 100644 --- a/libhdf5/hdf5debug.c +++ b/libhdf5/hdf5debug.c @@ -14,37 +14,9 @@ #ifdef H5CATCH -#define STSIZE 1000 - -#ifdef HAVE_EXECINFO_H -#ifdef H5BACKTRACE -# if !defined _WIN32 && !defined __CYGWIN__ -static void* stacktrace[STSIZE]; -# endif -#endif -#endif - int nch5breakpoint(int err) { -#ifdef HAVE_EXECINFO_H -#ifdef H5BACKTRACE -# if !defined _WIN32 && !defined __CYGWIN__ - int count = 0; - char** trace = NULL; - int i; - - count = backtrace(stacktrace,STSIZE); - trace = backtrace_symbols(stacktrace, STSIZE); - fprintf(stderr,"backtrace:\n"); - for(i=0;i Date: Sat, 4 Nov 2023 21:34:50 -0600 Subject: [PATCH 59/77] Update RELEASENOTES --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ee0d03cc94..75e335445f 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD -* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #2789](https://github.com/Unidata/netcdf-c/pull/2789). * Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764). * Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763). * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755). From fb62738f7284be7ced10d4e5d2ccd6880c53b013 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Mon, 6 Nov 2023 16:05:24 -0500 Subject: [PATCH 60/77] Renamed mmap variable, which conflicts with mmap() function on FreeBSD This fixes compiler errors on FreeBSD 14. --- libdispatch/dfile.c | 20 ++++++++++---------- nc_test/tst_diskless.c | 12 ++++++------ nc_test/tst_diskless3.c | 12 ++++++------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/libdispatch/dfile.c b/libdispatch/dfile.c index 7dba910f30..7447db3511 100644 --- a/libdispatch/dfile.c +++ b/libdispatch/dfile.c @@ -1770,7 +1770,7 @@ static int check_create_mode(int mode) { int mode_format; - int mmap = 0; + int use_mmap = 0; int inmemory = 0; int diskless = 0; @@ -1781,17 +1781,17 @@ check_create_mode(int mode) if (mode_format && (mode_format & (mode_format - 1))) return NC_EINVAL; - mmap = ((mode & NC_MMAP) == NC_MMAP); + use_mmap = ((mode & NC_MMAP) == NC_MMAP); inmemory = ((mode & NC_INMEMORY) == NC_INMEMORY); diskless = ((mode & NC_DISKLESS) == NC_DISKLESS); /* NC_INMEMORY and NC_DISKLESS and NC_MMAP are all mutually exclusive */ if(diskless && inmemory) return NC_EDISKLESS; - if(diskless && mmap) return NC_EDISKLESS; - if(inmemory && mmap) return NC_EINMEMORY; + if(diskless && use_mmap) return NC_EDISKLESS; + if(inmemory && use_mmap) return NC_EINMEMORY; /* mmap is not allowed for netcdf-4 */ - if(mmap && (mode & NC_NETCDF4)) return NC_EINVAL; + if(use_mmap && (mode & NC_NETCDF4)) return NC_EINVAL; #ifndef USE_NETCDF4 /* If the user asks for a netCDF-4 file, and the library was built @@ -1973,7 +1973,7 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, const NC_Dispatch* dispatcher = NULL; int inmemory = 0; int diskless = 0; - int mmap = 0; + int use_mmap = 0; char* path = NULL; NCmodel model; char* newpath = NULL; @@ -1989,17 +1989,17 @@ NC_open(const char *path0, int omode, int basepe, size_t *chunksizehintp, {stat = NC_EINVAL; goto done;} /* Capture the inmemory related flags */ - mmap = ((omode & NC_MMAP) == NC_MMAP); + use_mmap = ((omode & NC_MMAP) == NC_MMAP); diskless = ((omode & NC_DISKLESS) == NC_DISKLESS); inmemory = ((omode & NC_INMEMORY) == NC_INMEMORY); /* NC_INMEMORY and NC_DISKLESS and NC_MMAP are all mutually exclusive */ if(diskless && inmemory) {stat = NC_EDISKLESS; goto done;} - if(diskless && mmap) {stat = NC_EDISKLESS; goto done;} - if(inmemory && mmap) {stat = NC_EINMEMORY; goto done;} + if(diskless && use_mmap) {stat = NC_EDISKLESS; goto done;} + if(inmemory && use_mmap) {stat = NC_EINMEMORY; goto done;} /* mmap is not allowed for netcdf-4 */ - if(mmap && (omode & NC_NETCDF4)) {stat = NC_EINVAL; goto done;} + if(use_mmap && (omode & NC_NETCDF4)) {stat = NC_EINVAL; goto done;} /* Attempt to do file path conversion: note that this will do nothing if path is a 'file:...' url, so it will need to be diff --git a/nc_test/tst_diskless.c b/nc_test/tst_diskless.c index 13ae21b354..279685539a 100644 --- a/nc_test/tst_diskless.c +++ b/nc_test/tst_diskless.c @@ -46,7 +46,7 @@ void fail(int line) { #endif /* Control flags */ -static int flags, persist, usenetcdf4, mmap, diskless; +static int flags, persist, usenetcdf4, use_mmap, diskless; char* smode(int mode) @@ -90,13 +90,13 @@ main(int argc, char **argv) /* Set defaults */ persist = 0; usenetcdf4 = 0; - mmap = 0; + use_mmap = 0; diskless = 0; for(i=1;i Date: Tue, 7 Nov 2023 12:18:13 -0700 Subject: [PATCH 61/77] Added fenceposts so that dependencies and options specific to nczarr are skipped if nczarr is not enabled. --- configure.ac | 213 ++++++++++++++++++++++++++------------------------- 1 file changed, 110 insertions(+), 103 deletions(-) diff --git a/configure.ac b/configure.ac index 19386fe141..a35fd764d2 100644 --- a/configure.ac +++ b/configure.ac @@ -857,129 +857,136 @@ AC_MSG_RESULT([${have_sz}]) ########## -# See if we have libzip for NCZarr -AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no]) -AC_MSG_CHECKING([whether libzip library is available]) -AC_MSG_RESULT([${have_zip}]) -enable_nczarr_zip=${have_zip} # alias +## +# Check to see if we're using NCZarr. If not, we don't need to check for dependencies and such. +## if test "x$enable_nczarr" = xno ; then enable_nczarr_zip=no -fi +else -AC_MSG_CHECKING([whether nczarr zip support is enabled]) -AC_MSG_RESULT([${enable_nczarr_zip}]) + # See if we have libzip for NCZarr + AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no]) + AC_MSG_CHECKING([whether libzip library is available]) + AC_MSG_RESULT([${have_zip}]) + enable_nczarr_zip=${have_zip} # alias -if test "x$enable_nczarr_zip" = xyes ; then -AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) -fi -# Check for enabling of S3 support -AC_MSG_CHECKING([whether netcdf S3 support should be enabled]) -AC_ARG_ENABLE([s3], - [AS_HELP_STRING([--enable-s3], - [enable netcdf S3 support])]) -test "x$enable_s3" = xyes || enable_s3=no -AC_MSG_RESULT($enable_s3) -if test "x$enable_remote_functionality" = xno ; then - AC_MSG_WARN([--disable-remote-functionality => --disable-s3]) - enable_s3=no -fi - -# --enable-nczarr-s3 is synonym for --enable-s3 (but...) -AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) -AC_ARG_ENABLE([nczarr-s3], - [AS_HELP_STRING([--enable-nczarr-s3], - [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) -AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)]) - -# Set enable_s3 instead of enable_nczarr_s3 -if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then -enable_s3=yes # back compatibility -fi -unset enable_nczarr_s3 - -# Note we check for the library after checking for enable_s3 -# because for some reason this fails if we unconditionally test for sdk -# and it is not available. Fix someday -S3LIBS="" -if test "x$enable_s3" = xyes ; then -# See if we have the s3 aws library -# Check for the AWS S3 SDK library - AC_LANG_PUSH([C++]) - AC_CHECK_LIB([aws-c-common], [aws_string_destroy], [enable_s3_aws=yes],[enable_s3_aws=no]) - if test "x$enable_s3_aws" = "xyes" ; then - S3LIBS="-laws-cpp-sdk-core -laws-cpp-sdk-s3" + AC_MSG_CHECKING([whether nczarr zip support is enabled]) + AC_MSG_RESULT([${enable_nczarr_zip}]) + + if test "x$enable_nczarr_zip" = xyes ; then + AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) fi - AC_LANG_POP -else - enable_s3_aws=no -fi -AC_MSG_CHECKING([whether AWS S3 SDK library is available]) -AC_MSG_RESULT([$enable_s3_aws]) + # Check for enabling of S3 support + AC_MSG_CHECKING([whether netcdf S3 support should be enabled]) + AC_ARG_ENABLE([s3], + [AS_HELP_STRING([--enable-s3], + [enable netcdf S3 support])]) + test "x$enable_s3" = xyes || enable_s3=no + AC_MSG_RESULT($enable_s3) + + if test "x$enable_remote_functionality" = xno ; then + AC_MSG_WARN([--disable-remote-functionality => --disable-s3]) + enable_s3=no + fi -# Check for enabling forced use of Internal S3 library -AC_MSG_CHECKING([whether internal S3 support should be used]) -AC_ARG_ENABLE([s3-internal], - [AS_HELP_STRING([--enable-s3-internal], - [enable internal S3 support])]) -test "x$enable_s3_internal" = xyes || enable_s3_internal=no -AC_MSG_RESULT($enable_s3_internal) + # --enable-nczarr-s3 is synonym for --enable-s3 (but...) + AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) + AC_ARG_ENABLE([nczarr-s3], + [AS_HELP_STRING([--enable-nczarr-s3], + [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) + AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)]) -if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then -AC_MSG_WARN([No S3 library available => S3 support disabled]) -enable_s3=no -fi + # Set enable_s3 instead of enable_nczarr_s3 + if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then + enable_s3=yes # back compatibility + fi + unset enable_nczarr_s3 + + # Note we check for the library after checking for enable_s3 + # because for some reason this fails if we unconditionally test for sdk + # and it is not available. Fix someday + S3LIBS="" + if test "x$enable_s3" = xyes ; then + # See if we have the s3 aws library + # Check for the AWS S3 SDK library + AC_LANG_PUSH([C++]) + AC_CHECK_LIB([aws-c-common], [aws_string_destroy], [enable_s3_aws=yes],[enable_s3_aws=no]) + if test "x$enable_s3_aws" = "xyes" ; then + S3LIBS="-laws-cpp-sdk-core -laws-cpp-sdk-s3" + fi + AC_LANG_POP + else + enable_s3_aws=no + fi -if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then -AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.]) -enable_s3_aws=no -fi + AC_MSG_CHECKING([whether AWS S3 SDK library is available]) + AC_MSG_RESULT([$enable_s3_aws]) -if test "x$enable_s3_internal" = xyes ; then -if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then -# Find crypto libraries if using ssl -AC_CHECK_LIB([ssl],[ssl_create_cipher_list]) -AC_CHECK_LIB([crypto],[SHA256]) -fi -fi + # Check for enabling forced use of Internal S3 library + AC_MSG_CHECKING([whether internal S3 support should be used]) + AC_ARG_ENABLE([s3-internal], + [AS_HELP_STRING([--enable-s3-internal], + [enable internal S3 support])]) + test "x$enable_s3_internal" = xyes || enable_s3_internal=no + AC_MSG_RESULT($enable_s3_internal) -# Check for enabling S3 testing -AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled]) -AC_ARG_WITH([s3-testing], - [AS_HELP_STRING([--with-s3-testing=yes|no|public], - [control netcdf S3 testing])], - [], [with_s3_testing=public]) -AC_MSG_RESULT($with_s3_testing) + if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then + AC_MSG_WARN([No S3 library available => S3 support disabled]) + enable_s3=no + fi -# Disable S3 tests if S3 support is disabled -if test "x$enable_s3" = xno ; then - if test "x$with_s3_testing" != xno ; then - AC_MSG_WARN([S3 support is disabled => no testing]) - with_s3_testing=no - fi -fi + if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then + AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.]) + enable_s3_aws=no + fi -if test "x$enable_s3" = xyes ; then - AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled]) -fi + if test "x$enable_s3_internal" = xyes ; then + if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then + # Find crypto libraries if using ssl + AC_CHECK_LIB([ssl],[ssl_create_cipher_list]) + AC_CHECK_LIB([crypto],[SHA256]) + fi + fi -if test "x$enable_s3_aws" = xyes ; then - LIBS="$LIBS$S3LIBS" - AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library]) -fi + # Check for enabling S3 testing + AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled]) + AC_ARG_WITH([s3-testing], + [AS_HELP_STRING([--with-s3-testing=yes|no|public], + [control netcdf S3 testing])], + [], [with_s3_testing=public]) + AC_MSG_RESULT($with_s3_testing) + + # Disable S3 tests if S3 support is disabled + if test "x$enable_s3" = xno ; then + if test "x$with_s3_testing" != xno ; then + AC_MSG_WARN([S3 support is disabled => no testing]) + with_s3_testing=no + fi + fi -if test "x$enable_s3_internal" = xyes ; then - AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library]) -fi + if test "x$enable_s3" = xyes ; then + AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled]) + fi -AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.]) + if test "x$enable_s3_aws" = xyes ; then + LIBS="$LIBS$S3LIBS" + AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library]) + fi -if test "x$with_s3_testing" = xyes ; then - AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) - AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.]) + if test "x$enable_s3_internal" = xyes ; then + AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library]) + fi + + AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.]) + + if test "x$with_s3_testing" = xyes ; then + AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) + AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.]) + fi fi # Check whether we want to enable strict null byte header padding. From 99d0760c0defab221f89659ab43563a3a7da36c2 Mon Sep 17 00:00:00 2001 From: Tobias Bussmann Date: Thu, 9 Nov 2023 00:39:16 +0100 Subject: [PATCH 62/77] disable test that depend on ncpathcvt in cmake build w/o utilities fixes #2794 --- nctest/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/nctest/CMakeLists.txt b/nctest/CMakeLists.txt index d7de84a6cd..ada220f2ab 100644 --- a/nctest/CMakeLists.txt +++ b/nctest/CMakeLists.txt @@ -18,7 +18,9 @@ TARGET_LINK_LIBRARIES(nctest netcdf) ADD_TEST(nctest ${EXECUTABLE_OUTPUT_PATH}/nctest) add_bin_test_no_prefix(tst_rename) -add_sh_test(nctest compare_test_files) -IF(HAVE_BASH) - SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) -ENDIF(HAVE_BASH) +IF(BUILD_UTILITIES) + add_sh_test(nctest compare_test_files) + IF(HAVE_BASH) + SET_TESTS_PROPERTIES(nctest_compare_test_files PROPERTIES DEPENDS nctest) + ENDIF(HAVE_BASH) +ENDIF(BUILD_UTILITIES) From ce33d888a11d1bd38b2c03904e3891988e590ce6 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 14 Nov 2023 15:43:58 -0700 Subject: [PATCH 63/77] Add missing ERANGE_FILL to cmake-generated config.h. Include config.h in ncx.c. --- config.h.cmake.in | 6 ++++++ libsrc/ncx.m4 | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config.h.cmake.in b/config.h.cmake.in index 04d876ff17..941240390a 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -118,6 +118,12 @@ are set when opening a binary file on Windows. */ /* if true, build byte-range Client */ #cmakedefine ENABLE_BYTERANGE 1 +/* if true, enable ERANGE fill */ +#cmakedefine ENABLE_ERANGE_FILL 1 +#ifdef ENABLE_ERANGE_FILL +#define ERANGE_FILL 1 +#endif + /* if true, use hdf5 S3 virtual file reader */ #cmakedefine ENABLE_HDF5_ROS3 1 diff --git a/libsrc/ncx.m4 b/libsrc/ncx.m4 index c8ea6df416..48b8cf4ad2 100644 --- a/libsrc/ncx.m4 +++ b/libsrc/ncx.m4 @@ -39,7 +39,7 @@ ifdef(`PNETCDF',` `#'if HAVE_CONFIG_H `#'include `#'endif') - +#include #include #include #include From 374d65464452284e6e88736d12523cb6cae38895 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Sun, 30 Apr 2023 18:00:35 -0400 Subject: [PATCH 64/77] Fixed misaligned memory access flagged by UBSan Use memcpy to copy correctly even for unaligned memory. This was already done for some functions here, but not all. Also took the oppurtunity to remove a bunch of seemingly obsolete/commented code. --- libsrc/ncx.m4 | 268 ++++---------------------------------------------- 1 file changed, 17 insertions(+), 251 deletions(-) diff --git a/libsrc/ncx.m4 b/libsrc/ncx.m4 index c8ea6df416..a9fb11f41b 100644 --- a/libsrc/ncx.m4 +++ b/libsrc/ncx.m4 @@ -307,88 +307,24 @@ swapn2b(void *dst, const void *src, IntType nn) IntType i; uint16_t *op = (uint16_t*) dst; uint16_t *ip = (uint16_t*) src; + uint16_t tmp; for (i=0; i 0) - * { - * *op++ = *(++ip); - * *op++ = *(ip++ -1); - * } - */ - while (nn > 3) - { - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - *op++ = *(++ip); - *op++ = *(ip++ -1); - nn -= 4; - } - while (nn-- > 0) - { - *op++ = *(++ip); - *op++ = *(ip++ -1); - } -#endif } # ifndef vax inline static void swap4b(void *dst, const void *src) { - /* copy over, make the below swap in-place */ uint32_t tmp; - /* use memcpy to avoid type punning */ + /* memcpy is used to handle the case of unaligned memory */ memcpy(&tmp, src, sizeof(tmp)); tmp = SWAP4(tmp); memcpy(dst, &tmp, 4); - - /* Codes below will cause "break strict-aliasing rules" in gcc - uint32_t *op = (uint32_t*)dst; - *op = *(uint32_t*)src; - *op = SWAP4(*op); - */ - - /* Below are copied from netCDF-4. - * See https://bugtracking.unidata.ucar.edu/browse/NCF-338 - * Quote "One issue we are wrestling with is how compilers optimize this - * code. For some reason, we are actually needing to add an artificial - * move to a 4 byte space to get it to work. I think what is happening is - * that the optimizer is bit shifting within a double, which is incorrect. - * The following code actually does work correctly. - * This is in Linux land, gcc. - * - * However, the above in-place byte-swap does not appear affected by this. - */ -#if 0 - uint32_t *ip = (uint32_t*)src; - uint32_t tempOut; /* cannot use pointer when gcc O2 optimizer is used */ - tempOut = SWAP4(*ip); - - *(float *)dst = *(float *)(&tempOut); -#endif - - /* OLD implementation that results in four load and four store CPU - instructions - char *op = dst; - const char *ip = src; - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - */ - } # endif /* !vax */ @@ -398,110 +334,24 @@ swapn4b(void *dst, const void *src, IntType nn) IntType i; uint32_t *op = (uint32_t*) dst; uint32_t *ip = (uint32_t*) src; + uint32_t tmp; for (i=0; i 0) - * { - * op[0] = ip[3]; - * op[1] = ip[2]; - * op[2] = ip[1]; - * op[3] = ip[0]; - * op += 4; - * ip += 4; - * } - */ - while (nn > 3) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; - op[8] = ip[11]; - op[9] = ip[10]; - op[10] = ip[9]; - op[11] = ip[8]; - op[12] = ip[15]; - op[13] = ip[14]; - op[14] = ip[13]; - op[15] = ip[12]; - op += 16; - ip += 16; - nn -= 4; - } - while (nn-- > 0) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op += 4; - ip += 4; - } -#endif } # ifndef vax inline static void swap8b(void *dst, const void *src) { -#ifdef FLOAT_WORDS_BIGENDIAN - /* copy over, make the below swap in-place */ - *(uint64_t*)dst = *(uint64_t*)src; - - uint32_t *op = (uint32_t*)dst; - *op = SWAP4(*op); - op = (uint32_t*)((char*)dst+4); - *op = SWAP4(*op); -#else uint64_t tmp; - /* use memcpy to avoid type punning */ + /* memcpy is used to handle the case of unaligned memory */ memcpy(&tmp, src, sizeof(tmp)); tmp = SWAP8(tmp); memcpy(dst, &tmp, 8); - - /* Codes below will cause "break strict-aliasing rules" in gcc - uint64_t *op = (uint64_t*)dst; - *op = *(uint64_t*)src; - *op = SWAP8(*op); - */ -#endif - -#if 0 - char *op = dst; - const char *ip = src; -# ifndef FLOAT_WORDS_BIGENDIAN - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; -# else - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; -#endif -#endif } # endif /* !vax */ @@ -509,100 +359,16 @@ swap8b(void *dst, const void *src) inline static void swapn8b(void *dst, const void *src, IntType nn) { -#ifdef FLOAT_WORDS_BIGENDIAN - IntType i; - uint64_t *dst_p = (uint64_t*) dst; - uint64_t *src_p = (uint64_t*) src; - for (i=0; i 0) - * { - * op[0] = ip[7]; - * op[1] = ip[6]; - * op[2] = ip[5]; - * op[3] = ip[4]; - * op[4] = ip[3]; - * op[5] = ip[2]; - * op[6] = ip[1]; - * op[7] = ip[0]; - * op += 8; - * ip += 8; - * } - */ -# ifndef FLOAT_WORDS_BIGENDIAN - while (nn > 1) - { - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; - op[8] = ip[15]; - op[9] = ip[14]; - op[10] = ip[13]; - op[11] = ip[12]; - op[12] = ip[11]; - op[13] = ip[10]; - op[14] = ip[9]; - op[15] = ip[8]; - op += 16; - ip += 16; - nn -= 2; - } - while (nn-- > 0) - { - op[0] = ip[7]; - op[1] = ip[6]; - op[2] = ip[5]; - op[3] = ip[4]; - op[4] = ip[3]; - op[5] = ip[2]; - op[6] = ip[1]; - op[7] = ip[0]; - op += 8; - ip += 8; - } -# else - while (nn-- > 0) - { - op[0] = ip[3]; - op[1] = ip[2]; - op[2] = ip[1]; - op[3] = ip[0]; - op[4] = ip[7]; - op[5] = ip[6]; - op[6] = ip[5]; - op[7] = ip[4]; - op += 8; - ip += 8; - } -#endif -#endif } # endif /* !vax */ From dc1b8b9c4b4880d3060c519a1fdc7680c31142dc Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Tue, 14 Nov 2023 22:28:01 -0500 Subject: [PATCH 65/77] Fixed various UBSan warnings about working with NULL pointers Any pointer arithmetic with NULL pointers is technically UB, even if you don't end up dereferencing the pointer. --- libsrc/attr.m4 | 17 +++++---- libsrc/dim.c | 17 +++++---- libsrc/nc3internal.c | 17 +++++---- libsrc/putget.m4 | 2 +- libsrc/v1hpg.c | 84 ++++++++++++++++++++++++++------------------ libsrc/var.c | 15 ++++---- 6 files changed, 90 insertions(+), 62 deletions(-) diff --git a/libsrc/attr.m4 b/libsrc/attr.m4 index 94bebf2386..dba0238e18 100644 --- a/libsrc/attr.m4 +++ b/libsrc/attr.m4 @@ -230,15 +230,18 @@ dup_NC_attrarrayV(NC_attrarray *ncap, const NC_attrarray *ref) ncap->nelems = 0; { NC_attr **app = ncap->value; - const NC_attr **drpp = (const NC_attr **)ref->value; - NC_attr *const *const end = &app[ref->nelems]; - for( /*NADA*/; app < end; drpp++, app++, ncap->nelems++) + if (app) { - *app = dup_NC_attr(*drpp); - if(*app == NULL) + const NC_attr **drpp = (const NC_attr **)ref->value; + NC_attr *const *const end = &app[ref->nelems]; + for( /*NADA*/; app < end; drpp++, app++, ncap->nelems++) { - status = NC_ENOMEM; - break; + *app = dup_NC_attr(*drpp); + if(*app == NULL) + { + status = NC_ENOMEM; + break; + } } } } diff --git a/libsrc/dim.c b/libsrc/dim.c index 227ac5ab5c..dea0fac3dc 100644 --- a/libsrc/dim.c +++ b/libsrc/dim.c @@ -228,15 +228,18 @@ dup_NC_dimarrayV(NC_dimarray *ncap, const NC_dimarray *ref) ncap->nelems = 0; { NC_dim **dpp = ncap->value; - const NC_dim **drpp = (const NC_dim **)ref->value; - NC_dim *const *const end = &dpp[ref->nelems]; - for( /*NADA*/; dpp < end; drpp++, dpp++, ncap->nelems++) + if(dpp != NULL) { - *dpp = dup_NC_dim(*drpp); - if(*dpp == NULL) + const NC_dim **drpp = (const NC_dim **)ref->value; + NC_dim *const *const end = &dpp[ref->nelems]; + for( /*NADA*/; dpp < end; drpp++, dpp++, ncap->nelems++) { - status = NC_ENOMEM; - break; + *dpp = dup_NC_dim(*drpp); + if(*dpp == NULL) + { + status = NC_ENOMEM; + break; + } } } } diff --git a/libsrc/nc3internal.c b/libsrc/nc3internal.c index 2045304e05..0916cc9500 100644 --- a/libsrc/nc3internal.c +++ b/libsrc/nc3internal.c @@ -950,7 +950,6 @@ int NC_calcsize(const NC3_INFO *ncp, off_t *calcsizep) { NC_var **vpp = (NC_var **)ncp->vars.value; - NC_var *const *const end = &vpp[ncp->vars.nelems]; NC_var *last_fix = NULL; /* last "non-record" var */ int numrecvars = 0; /* number of record variables */ @@ -960,12 +959,16 @@ NC_calcsize(const NC3_INFO *ncp, off_t *calcsizep) return NC_NOERR; } - for( /*NADA*/; vpp < end; vpp++) { - if(IS_RECVAR(*vpp)) { - numrecvars++; - } else { - last_fix = *vpp; - } + if (vpp) + { + NC_var *const *const end = &vpp[ncp->vars.nelems]; + for( /*NADA*/; vpp < end; vpp++) { + if(IS_RECVAR(*vpp)) { + numrecvars++; + } else { + last_fix = *vpp; + } + } } if(numrecvars == 0) { diff --git a/libsrc/putget.m4 b/libsrc/putget.m4 index 2779480bab..9a0aca8a27 100644 --- a/libsrc/putget.m4 +++ b/libsrc/putget.m4 @@ -551,7 +551,7 @@ static int NCedgeck(const NC3_INFO* ncp, const NC_var *varp, const size_t *start, const size_t *edges) { - const size_t *const end = start + varp->ndims; + const size_t *const end = start ? (start + varp->ndims) : NULL; const size_t *shp = varp->shape; if(varp->ndims == 0) diff --git a/libsrc/v1hpg.c b/libsrc/v1hpg.c index 63172f2274..f93b219bdb 100644 --- a/libsrc/v1hpg.c +++ b/libsrc/v1hpg.c @@ -448,10 +448,13 @@ ncx_len_NC_dimarray(const NC_dimarray *ncap, int version) /* else */ { const NC_dim **dpp = (const NC_dim **)ncap->value; - const NC_dim *const *const end = &dpp[ncap->nelems]; - for( /*NADA*/; dpp < end; dpp++) + if (dpp) { - xlen += ncx_len_NC_dim(*dpp,version); + const NC_dim *const *const end = &dpp[ncap->nelems]; + for( /*NADA*/; dpp < end; dpp++) + { + xlen += ncx_len_NC_dim(*dpp,version); + } } } return xlen; @@ -641,11 +644,13 @@ v1h_put_NC_attrV(v1hs *psp, const NC_attr *attrp) if(status != NC_NOERR) return status; - (void) memcpy(psp->pos, value, nbytes); - + if (value) { + (void) memcpy(psp->pos, value, nbytes); + value = (void *)((char *)value + nbytes); + } + psp->pos = (void *)((char *)psp->pos + nbytes); - value = (void *)((char *)value + nbytes); - remaining -= nbytes; + remaining -= nbytes; } while(remaining != 0); @@ -709,11 +714,13 @@ v1h_get_NC_attrV(v1hs *gsp, NC_attr *attrp) if(status != NC_NOERR) return status; - (void) memcpy(value, gsp->pos, nget); + if (value) { + (void) memcpy(value, gsp->pos, nget); + value = (void *)((signed char *)value + nget); + } + gsp->pos = (void*)((unsigned char *)gsp->pos + nget); - value = (void *)((signed char *)value + nget); - remaining -= nget; } while(remaining != 0); @@ -790,10 +797,13 @@ ncx_len_NC_attrarray(const NC_attrarray *ncap, int version) /* else */ { const NC_attr **app = (const NC_attr **)ncap->value; - const NC_attr *const *const end = &app[ncap->nelems]; - for( /*NADA*/; app < end; app++) + if (app) { - xlen += ncx_len_NC_attr(*app,version); + const NC_attr *const *const end = &app[ncap->nelems]; + for( /*NADA*/; app < end; app++) + { + xlen += ncx_len_NC_attr(*app,version); + } } } return xlen; @@ -1090,10 +1100,13 @@ ncx_len_NC_vararray(const NC_vararray *ncap, size_t sizeof_off_t, int version) /* else */ { const NC_var **vpp = (const NC_var **)ncap->value; - const NC_var *const *const end = &vpp[ncap->nelems]; - for( /*NADA*/; vpp < end; vpp++) + if (vpp) { - xlen += ncx_len_NC_var(*vpp, sizeof_off_t, version); + const NC_var *const *const end = &vpp[ncap->nelems]; + for( /*NADA*/; vpp < end; vpp++) + { + xlen += ncx_len_NC_var(*vpp, sizeof_off_t, version); + } } } return xlen; @@ -1224,7 +1237,6 @@ static int NC_computeshapes(NC3_INFO* ncp) { NC_var **vpp = (NC_var **)ncp->vars.value; - NC_var *const *const end = &vpp[ncp->vars.nelems]; NC_var *first_var = NULL; /* first "non-record" var */ NC_var *first_rec = NULL; /* first "record" var */ int status; @@ -1236,27 +1248,31 @@ NC_computeshapes(NC3_INFO* ncp) if(ncp->vars.nelems == 0) return(0); - for( /*NADA*/; vpp < end; vpp++) + if (vpp) { - status = NC_var_shape(*vpp, &ncp->dims); - if(status != NC_NOERR) - return(status); - - if(IS_RECVAR(*vpp)) - { - if(first_rec == NULL) - first_rec = *vpp; - ncp->recsize += (*vpp)->len; - } - else + NC_var *const *const end = &vpp[ncp->vars.nelems]; + for( /*NADA*/; vpp < end; vpp++) { + status = NC_var_shape(*vpp, &ncp->dims); + if(status != NC_NOERR) + return(status); + + if(IS_RECVAR(*vpp)) + { + if(first_rec == NULL) + first_rec = *vpp; + ncp->recsize += (*vpp)->len; + } + else + { if(first_var == NULL) first_var = *vpp; - /* - * Overwritten each time thru. - * Usually overwritten in first_rec != NULL clause below. - */ - ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; + /* + * Overwritten each time thru. + * Usually overwritten in first_rec != NULL clause below. + */ + ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; + } } } diff --git a/libsrc/var.c b/libsrc/var.c index f09bbe27bd..c8cd64dcd6 100644 --- a/libsrc/var.c +++ b/libsrc/var.c @@ -267,14 +267,17 @@ dup_NC_vararrayV(NC_vararray *ncap, const NC_vararray *ref) { NC_var **vpp = ncap->value; const NC_var **drpp = (const NC_var **)ref->value; - NC_var *const *const end = &vpp[ref->nelems]; - for( /*NADA*/; vpp < end; drpp++, vpp++, ncap->nelems++) + if (vpp) { - *vpp = dup_NC_var(*drpp); - if(*vpp == NULL) + NC_var *const *const end = &vpp[ref->nelems]; + for( /*NADA*/; vpp < end; drpp++, vpp++, ncap->nelems++) { - status = NC_ENOMEM; - break; + *vpp = dup_NC_var(*drpp); + if(*vpp == NULL) + { + status = NC_ENOMEM; + break; + } } } } From 23aa46fcf51437de23a83862427acb605a8a8740 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 15 Nov 2023 15:11:56 -0500 Subject: [PATCH 66/77] Fixed various UBSan warnings about working with NULL pointers Any pointer arithmetic with NULL pointers is technically UB, even if you don't end up dereferencing the pointer. --- libdispatch/utf8proc.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libdispatch/utf8proc.c b/libdispatch/utf8proc.c index 3a05dd3b34..68d8dd4a20 100644 --- a/libdispatch/utf8proc.c +++ b/libdispatch/utf8proc.c @@ -355,7 +355,8 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_ for (; len >= 0; entry++, len--) { nc_utf8proc_int32_t entry_cp = nc_seqindex_decode_entry(&entry); - written += nc_utf8proc_decompose_char(entry_cp, dst+written, + nc_utf8proc_int32_t *dest = dst ? (dst+written) : NULL; + written += nc_utf8proc_decompose_char(entry_cp, dest, (bufsize > written) ? (bufsize - written) : 0, options, last_boundclass); if (written < 0) return UTF8PROC_ERROR_OVERFLOW; @@ -525,8 +526,10 @@ static nc_utf8proc_ssize_t nc_seqindex_write_char_decomposed(nc_utf8proc_uint16_ if (custom_func != NULL) { uc = custom_func(uc, custom_data); /* user-specified custom mapping */ } + nc_utf8proc_int32_t *dest = NULL; + if (buffer) dest = buffer + wpos; decomp_result = nc_utf8proc_decompose_char( - uc, buffer + wpos, (bufsize > wpos) ? (bufsize - wpos) : 0, options, + uc, dest, (bufsize > wpos) ? (bufsize - wpos) : 0, options, &boundclass ); if (decomp_result < 0) return decomp_result; From 58dd53022fe9c4ead056246964ddbde46c193626 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 27 Nov 2023 18:46:10 -0700 Subject: [PATCH 67/77] Fix some important bugs in various files The most critical bug is in nch5s3comms.c. I for some reason assumed that signing keys did not contain any zero bytes. But obviously it can, so a test was removed. Other fixes: 1. Guarantee allocated memory is initialized to all zeros. 2. Cleanup errmsg handling in libncpoco. 3. Fix processing of aws list-objects-v2 because I misread the syntax. --- include/ncpathmgr.h | 3 +++ include/ncs3sdk.h | 3 +++ libdispatch/dcopy.c | 4 ++-- libdispatch/derror.c | 4 ++-- libdispatch/nch5s3comms.c | 4 ++-- libdispatch/nclist.c | 7 +++++-- libncpoco/cp_unix.c | 2 +- libncpoco/cp_win32.c | 5 ++--- libncpoco/ncpoco.c | 11 +++++++++++ libncpoco/ncpoco.h | 3 +++ libsrc/v1hpg.c | 14 +++++++------- s3gc.in | 4 +++- 12 files changed, 44 insertions(+), 20 deletions(-) diff --git a/include/ncpathmgr.h b/include/ncpathmgr.h index 3f0bcd0ec0..91dd38ff02 100644 --- a/include/ncpathmgr.h +++ b/include/ncpathmgr.h @@ -14,6 +14,9 @@ #ifdef HAVE_DIRENT_H #include #endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif #ifdef HAVE_SYS_STAT_H #include #endif diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index 771faa6666..013710ccfd 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -6,6 +6,9 @@ #ifndef NCS3SDK_H #define NCS3SDK_H 1 +#define AWSHOST ".amazonaws.com" +#define GOOGLEHOST "storage.googleapis.com" + /* Track the server type, if known */ typedef enum NCS3SVC {NCS3UNK=0, /* unknown */ NCS3=1, /* s3.amazon.aws */ diff --git a/libdispatch/dcopy.c b/libdispatch/dcopy.c index 2491a92b6d..1f0ff034aa 100644 --- a/libdispatch/dcopy.c +++ b/libdispatch/dcopy.c @@ -694,7 +694,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2) int gid; uintptr_t id; - id = grp; + id = (uintptr_t)grp; nclistpush(queue,(void*)id); /* prime the queue */ while(nclistlength(queue) > 0) { id = (uintptr_t)nclistremove(queue,0); @@ -712,7 +712,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2) goto done; /* push onto the end of the queue */ for(i=0;isigning_key = signing_key; signing_key = NULL; @@ -2033,7 +2033,7 @@ NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *re if ((size_t)ret != (AWS4_secret_len - 1)) HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "problem writing AWS4+secret `%s`", secret); - if((md = (unsigned char*)malloc(SHA256_DIGEST_LENGTH))==NULL) + if((md = (unsigned char*)calloc(1,SHA256_DIGEST_LENGTH))==NULL) HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for signing key ."); /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index f2c3f4d47b..5ed21753eb 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -31,7 +31,7 @@ NClist* nclistnew(void) ncinitialized = 1; } */ - l = (NClist*)malloc(sizeof(NClist)); + l = (NClist*)calloc(1,sizeof(NClist)); if(l) { l->alloc=0; l->length=0; @@ -286,10 +286,13 @@ nclistclone(const NClist* l, int deep) void* nclistextract(NClist* l) { - void* result = l->content; + void* result = NULL; + if(l) { + result = l->content; l->alloc = 0; l->length = 0; l->content = NULL; + } return result; } diff --git a/libncpoco/cp_unix.c b/libncpoco/cp_unix.c index 3ccea2ed03..7ff01f79ec 100755 --- a/libncpoco/cp_unix.c +++ b/libncpoco/cp_unix.c @@ -46,7 +46,7 @@ static void ncperr(const char* fcn, NCPSharedLib* lib) { const char* msg = dlerror(); - lib->err.msg[0] = '\0'; + memset(lib->err.msg,0,sizeof(lib->err.msg)); if(msg != NULL) { strlcat(lib->err.msg,fcn,sizeof(lib->err.msg)); strlcat(lib->err.msg,": ",sizeof(lib->err.msg)); diff --git a/libncpoco/cp_win32.c b/libncpoco/cp_win32.c index b4284ac101..896344f638 100755 --- a/libncpoco/cp_win32.c +++ b/libncpoco/cp_win32.c @@ -105,10 +105,9 @@ load(NCPSharedLib* lib , const char* path0, int flags) char* msg = NULL; FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, errcode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &msg, 0, NULL); - if(msg) { + memset(lib->err.msg,0,sizeof(lib->err.msg)); + if(msg) strncpy(lib->err.msg,msg,sizeof(lib->err.msg)); - } else - lib->err.msg[0] = '\0'; ret = NC_ENOTFOUND; goto ldone; } diff --git a/libncpoco/ncpoco.c b/libncpoco/ncpoco.c index 12d4b41eec..b12b2e0a9b 100755 --- a/libncpoco/ncpoco.c +++ b/libncpoco/ncpoco.c @@ -68,6 +68,7 @@ EXTERNL int ncpload(NCPSharedLib* lib, const char* path, int flags) { if(lib == NULL || path == NULL) return NC_EINVAL; + ncpclearerrmsg(lib); return lib->api.load(lib,path,flags); } @@ -75,6 +76,7 @@ EXTERNL int ncpunload(NCPSharedLib* lib) /* Unloads a shared library. */ { if(lib == NULL) return NC_EINVAL; + ncpclearerrmsg(lib); return lib->api.unload(lib); } @@ -93,6 +95,7 @@ EXTERNL void* ncpgetsymbol(NCPSharedLib* lib,const char* name) { if(lib == NULL) return NULL; + ncpclearerrmsg(lib); return lib->api.getsymbol(lib,name); } @@ -113,3 +116,11 @@ ncpgeterrmsg(NCPSharedLib* lib) if(lib == NULL) return NULL; return (lib->err.msg[0] == '\0' ? NULL : lib->err.msg); } + +/* Clear the last err msg. */ +EXTERNL void +ncpclearerrmsg(NCPSharedLib* lib) +{ + if(lib == NULL) return; + memset(lib->err.msg,0,sizeof(lib->err.msg)); +} diff --git a/libncpoco/ncpoco.h b/libncpoco/ncpoco.h index 43fb077bab..fc1e993b99 100755 --- a/libncpoco/ncpoco.h +++ b/libncpoco/ncpoco.h @@ -71,6 +71,9 @@ EXTERNL const char* ncpgetpath(NCPSharedLib*); /* Return last err msg */ EXTERNL const char* ncpgeterrmsg(NCPSharedLib* lib); +/* Clear the last err msg. */ +EXTERNL void ncpclearerrmsg(NCPSharedLib* lib); + EXTERNL const char* intstr(int err1); #endif /*NCPOCO_H*/ diff --git a/libsrc/v1hpg.c b/libsrc/v1hpg.c index f93b219bdb..d1c6684469 100644 --- a/libsrc/v1hpg.c +++ b/libsrc/v1hpg.c @@ -1254,7 +1254,7 @@ NC_computeshapes(NC3_INFO* ncp) for( /*NADA*/; vpp < end; vpp++) { status = NC_var_shape(*vpp, &ncp->dims); - if(status != NC_NOERR) + if(status != NC_NOERR) return(status); if(IS_RECVAR(*vpp)) @@ -1265,13 +1265,13 @@ NC_computeshapes(NC3_INFO* ncp) } else { - if(first_var == NULL) + if(first_var == NULL) first_var = *vpp; - /* - * Overwritten each time thru. - * Usually overwritten in first_rec != NULL clause below. - */ - ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; + /* + * Overwritten each time thru. + * Usually overwritten in first_rec != NULL clause below. + */ + ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; } } } diff --git a/s3gc.in b/s3gc.in index 612e371d7a..a63345c6c7 100755 --- a/s3gc.in +++ b/s3gc.in @@ -61,7 +61,9 @@ if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTRE fi aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys while read -r line; do - KEY=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` + KEY0=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` + # Strip off any leading '/' + KEY=`echo "$KEY0" | sed -e 's|^[/]*\(.*\)|\1|'` # Ignore keys that do not start with ${S3TESTSUBTREE} PREFIX=`echo "$KEY" | sed -e 's|\([^/]*\)/.*|\1|'` if test "x$PREFIX" = "x$S3TESTSUBTREE" ; then From dfc2ac729690ccc7d89ae657e5c574120476820e Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Sun, 30 Apr 2023 18:01:49 -0400 Subject: [PATCH 68/77] Replaced trivial uses of sprintf with snprintf In all these cases the size of the buffer can be computed with sizeof. --- h5_test/tst_h_compounds.c | 10 +-- h5_test/tst_h_dimscales.c | 2 +- h5_test/tst_h_dimscales4.c | 2 +- h5_test/tst_h_files.c | 6 +- h5_test/tst_h_mem.c | 2 +- h5_test/tst_h_vars.c | 2 +- h5_test/tst_h_vars2.c | 2 +- hdf4_test/tst_interops2.c | 2 +- libdap2/dapodom.c | 2 +- libdap2/dcetab.c | 2 +- libdap4/d4odom.c | 2 +- libdispatch/dparallel.c | 2 +- libdispatch/nctime.c | 2 +- libhdf5/hdf5open.c | 2 +- libhdf5/nc4hdf.c | 10 +-- libsrc/ffio.c | 2 +- libsrc/pstdint.h | 20 ++--- libsrc4/nc4internal.c | 4 +- nc_perf/bm_file.c | 4 +- nc_perf/bm_many_atts.c | 4 +- nc_perf/bm_many_objs.c | 6 +- nc_perf/tst_attsperf.c | 8 +- nc_perf/tst_bm_rando.c | 6 +- nc_perf/tst_chunks3.c | 36 ++++----- nc_perf/tst_compress.c | 4 +- nc_perf/tst_compress_par.c | 4 +- nc_perf/tst_create_files.c | 10 +-- nc_perf/tst_files2.c | 16 ++-- nc_perf/tst_h_many_atts.c | 2 +- nc_perf/tst_wrf_reads.c | 6 +- nc_test/large_files.c | 2 +- nc_test/quick_large_files.c | 2 +- nc_test/testnc3perf.c | 12 +-- nc_test/tst_addvar.c | 2 +- nc_test/tst_big_rvar.c | 2 +- nc_test/tst_big_var.c | 2 +- nc_test/tst_big_var2.c | 2 +- nc_test/tst_big_var6.c | 2 +- nc_test/tst_cdf5format.c | 6 +- nc_test/tst_diskless3.c | 2 +- nc_test/tst_diskless4.c | 2 +- nc_test/tst_formats.c | 8 +- nc_test/tst_large.c | 2 +- nc_test/tst_large_cdf5.c | 2 +- nc_test/tst_parallel2.c | 2 +- nc_test/tst_pnetcdf.c | 8 +- nc_test/tst_small.c | 4 +- nc_test/util.c | 6 +- nc_test4/tst_atts2.c | 2 +- nc_test4/tst_chunks.c | 8 +- nc_test4/tst_chunks2.c | 16 ++-- nc_test4/tst_compounds.c | 4 +- nc_test4/tst_compounds3.c | 4 +- nc_test4/tst_dims2.c | 2 +- nc_test4/tst_files.c | 2 +- nc_test4/tst_grps.c | 4 +- nc_test4/tst_interops4.c | 8 +- nc_test4/tst_interops5.c | 4 +- nc_test4/tst_large2.c | 2 +- nc_test4/tst_large3.c | 2 +- nc_test4/tst_large5.c | 2 +- nc_test4/tst_nc4perf.c | 8 +- nc_test4/tst_parallel.c | 2 +- nc_test4/tst_parallel3.c | 2 +- nc_test4/tst_parallel4.c | 2 +- nc_test4/tst_parallel_zlib.c | 4 +- nc_test4/tst_quantize.c | 6 +- nc_test4/tst_quantize_par.c | 4 +- nc_test4/tst_rename2.c | 12 +-- nc_test4/tst_rename3.c | 6 +- nc_test4/tst_simplerw_coll_r.c | 2 +- nc_test4/tst_strings.c | 2 +- nc_test4/tst_types.c | 4 +- nc_test4/tst_vars.c | 2 +- nc_test4/tst_vars3.c | 2 +- ncdump/ncvalidator.c | 10 +-- ncdump/ocprint.c | 4 +- ncgen/cmldata.c | 22 +++--- ncgen/cvt.c | 22 +++--- ncgen/dump.c | 24 +++--- ncgen/f77data.c | 14 ++-- ncgen/gencml.c | 2 +- ncgen/genjjni.c | 26 +++---- ncgen/jdatajni.c | 14 ++-- ncgen/ncgen.l | 24 +++--- ncgen/ncgen.y | 4 +- ncgen/ncgenl.c | 24 +++--- ncgen/ncgeny.c | 6 +- ncgen/odom.c | 2 +- ncgen3/genlib.c | 136 ++++++++++++++++----------------- ncgen3/load.c | 82 ++++++++++---------- ncgen3/ncgen.l | 14 ++-- ncgen3/ncgenl.c | 14 ++-- ncgen3/ncgeny.c | 2 +- nctest/driver.c | 2 +- nczarr_test/bm_chunks3.c | 4 +- nczarr_test/test_zchunks.c | 8 +- nczarr_test/test_zchunks2.c | 16 ++-- oc2/dapparse.c | 4 +- oc2/dapy.c | 2 +- oc2/ocdump.c | 18 ++--- 101 files changed, 434 insertions(+), 434 deletions(-) diff --git a/h5_test/tst_h_compounds.c b/h5_test/tst_h_compounds.c index c53032c3f9..952e2585a3 100644 --- a/h5_test/tst_h_compounds.c +++ b/h5_test/tst_h_compounds.c @@ -452,7 +452,7 @@ main() hr_data_out[i].starfleet_id = i; hr_data_out[i].svc_rec.i1 = 95; hr_data_out[i].svc_rec.i2 = 90; - if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; + if (snprintf(hr_data_out[i].name, sizeof(hr_data_out[i].name), "alien_%d", i) < 0) ERR; hr_data_out[i].max_temp = 99.99f; hr_data_out[i].min_temp = -9.99f; hr_data_out[i].percent_transporter_errosion = .1; @@ -557,7 +557,7 @@ main() hr_data_out[i].starfleet_id = i; hr_data_out[i].svc_rec.i1 = 95; hr_data_out[i].svc_rec.i2 = 90; - if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; + if (snprintf(hr_data_out[i].name, sizeof(hr_data_out[i].name), "alien_%d", i) < 0) ERR; hr_data_out[i].max_temp = 99.99f; hr_data_out[i].min_temp = -9.99f; hr_data_out[i].percent_transporter_errosion = .1; @@ -665,7 +665,7 @@ main() hr_data_out[i].starfleet_id = i; hr_data_out[i].svc_rec.i1 = 95; hr_data_out[i].svc_rec.i2 = 90; - if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; + if (snprintf(hr_data_out[i].name, sizeof(hr_data_out[i].name), "alien_%d", i) < 0) ERR; hr_data_out[i].max_temp = 99.99f; hr_data_out[i].min_temp = -9.99f; hr_data_out[i].percent_transporter_errosion = .1; @@ -756,7 +756,7 @@ main() /* Create some phony data. */ for (i = 0; i < DIM1_LEN; i++) { - if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; + if (snprintf(hr_data_out[i].name, sizeof(hr_data_out[i].name), "alien_%d", i) < 0) ERR; hr_data_out[i].max_temp = 99.99f; } @@ -841,7 +841,7 @@ main() /* Create some phony data. */ for (i = 0; i < DIM2_LEN; i++) { - if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; + if (snprintf(hr_data_out[i].name, sizeof(hr_data_out[i].name), "alien_%d", i) < 0) ERR; hr_data_out[i].max_temp = 99.99f; } diff --git a/h5_test/tst_h_dimscales.c b/h5_test/tst_h_dimscales.c index 7186778fe7..8eeee14c42 100644 --- a/h5_test/tst_h_dimscales.c +++ b/h5_test/tst_h_dimscales.c @@ -308,7 +308,7 @@ main() if ((var1_spaceid = H5Screate_simple(1, dims, dims)) < 0) ERR; for (v = 0; v < NUM_DATASETS; v++) { - sprintf(var_name, "var_%d", v); + snprintf(var_name, sizeof(var_name), "var_%d", v); if ((var1_datasetid[v] = H5Dcreate1(grpid, var_name, H5T_NATIVE_INT, var1_spaceid, H5P_DEFAULT)) < 0) ERR; if (H5DSattach_scale(var1_datasetid[v], dimscaleid, 0) < 0) ERR; diff --git a/h5_test/tst_h_dimscales4.c b/h5_test/tst_h_dimscales4.c index c12ca9624e..430960926b 100644 --- a/h5_test/tst_h_dimscales4.c +++ b/h5_test/tst_h_dimscales4.c @@ -93,7 +93,7 @@ main() spaceid, create_propid)) < 0) ERR; if (H5Sclose(spaceid) < 0) ERR; if (H5Pclose(create_propid) < 0) ERR; - sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, DIM_LEN); + snprintf(dimscale_wo_var, sizeof(dimscale_wo_var), "%s%10d", DIM_WITHOUT_VARIABLE, DIM_LEN); if (H5DSset_scale(dimscaleid, dimscale_wo_var) < 0) ERR; /* Create a variable that uses this dimension scale. */ diff --git a/h5_test/tst_h_files.c b/h5_test/tst_h_files.c index f4bc617f0c..15765ea5bd 100644 --- a/h5_test/tst_h_files.c +++ b/h5_test/tst_h_files.c @@ -248,7 +248,7 @@ main() void *bufr; void *fillp = NULL; - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE_NAME); /* Create file access and create property lists. */ if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) ERR; @@ -285,7 +285,7 @@ main() spaceid, plistid)) < 0) ERR; if (H5Sclose(spaceid) < 0) ERR; if (H5Pclose(plistid) < 0) ERR; - sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, DIM1); + snprintf(dimscale_wo_var, sizeof(dimscale_wo_var), "%s%10d", DIM_WITHOUT_VARIABLE, DIM1); if (H5DSset_scale(dim1_dimscaleid, dimscale_wo_var) < 0) ERR; /* Create the dim2 dimscale. */ @@ -299,7 +299,7 @@ main() spaceid, plistid)) < 0) ERR; if (H5Sclose(spaceid) < 0) ERR; if (H5Pclose(plistid) < 0) ERR; - sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, DIM2); + snprintf(dimscale_wo_var, sizeof(dimscale_wo_var), "%s%10d", DIM_WITHOUT_VARIABLE, DIM2); if (H5DSset_scale(dim2_dimscaleid, dimscale_wo_var) < 0) ERR; /* Now create the 2D dataset. */ diff --git a/h5_test/tst_h_mem.c b/h5_test/tst_h_mem.c index d2b1d4a242..d7b0b9e5d1 100644 --- a/h5_test/tst_h_mem.c +++ b/h5_test/tst_h_mem.c @@ -67,7 +67,7 @@ main() /* Create the variables. */ for (v = 0; v < NUM_DATASETS; v++) { - sprintf(var_name, "var_%d", v); + snprintf(var_name, sizeof(var_name), "var_%d", v); /* printf("creating var %s\n", var_name);*/ if ((datasetid[v] = H5Dcreate1(grpid, var_name, H5T_NATIVE_INT, spaceid, plistid)) < 0) ERR_RET; diff --git a/h5_test/tst_h_vars.c b/h5_test/tst_h_vars.c index 77be6abd4f..7fa840f412 100644 --- a/h5_test/tst_h_vars.c +++ b/h5_test/tst_h_vars.c @@ -517,7 +517,7 @@ main() "szip_and_zlib"}; /* Open file and create group. */ - sprintf(file_name, "%s_%s.h5", TEST_NAME, desc[f]); + snprintf(file_name, sizeof(file_name), "%s_%s.h5", TEST_NAME, desc[f]); if ((fileid = H5Fcreate(file_name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) ERR; if ((grpid = H5Gcreate1(fileid, GRP_NAME, 0)) < 0) ERR; diff --git a/h5_test/tst_h_vars2.c b/h5_test/tst_h_vars2.c index 400be6bb49..e53b677529 100644 --- a/h5_test/tst_h_vars2.c +++ b/h5_test/tst_h_vars2.c @@ -478,7 +478,7 @@ main() /* Create the variables. */ for (v = 0; v < NUM_DATASETS; v++) { - sprintf(var_name, "var_%d", v); + snprintf(var_name, sizeof(var_name), "var_%d", v); /* printf("creating var %s\n", var_name);*/ if ((datasetid[v] = H5Dcreate1(grpid, var_name, H5T_NATIVE_INT, spaceid, plistid)) < 0) ERR_RET; diff --git a/hdf4_test/tst_interops2.c b/hdf4_test/tst_interops2.c index 2b7ff2681a..7951dc5e0e 100644 --- a/hdf4_test/tst_interops2.c +++ b/hdf4_test/tst_interops2.c @@ -141,7 +141,7 @@ main(int argc, char **argv) /* Create some HDF4 datasets. */ for (t = 0; t < NUM_TYPES; t++) { - sprintf(tmp_name, "hdf4_dataset_type_%d", t); + snprintf(tmp_name, sizeof(tmp_name), "hdf4_dataset_type_%d", t); if ((sds_id = SDcreate(sd_id, tmp_name, hdf4_type[t], DIMS_3, dim_size)) == FAIL) ERR; /* Set up dimensions. By giving them the same names for each diff --git a/libdap2/dapodom.c b/libdap2/dapodom.c index 2c6611a98d..c8d4a22396 100644 --- a/libdap2/dapodom.c +++ b/libdap2/dapodom.c @@ -81,7 +81,7 @@ dapodom_print(Dapodometer* odom) if(odom->rank == 0) { strlcat(line,"[]",sizeof(line)); } else for(i=0;irank;i++) { - sprintf(tmp,"[%lu/%lu:%lu:%lu]", + snprintf(tmp,sizeof(tmp),"[%lu/%lu:%lu:%lu]", (size_t)odom->index[i], (size_t)odom->start[i], (size_t)odom->stride[i], diff --git a/libdap2/dcetab.c b/libdap2/dcetab.c index 60a602449d..4b4adf465b 100644 --- a/libdap2/dcetab.c +++ b/libdap2/dcetab.c @@ -978,7 +978,7 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, return 1; } - /* Avoid sprintf, as that infringes on the user's name space. + /* Avoid snprintf, as that infringes on the user's name space. Don't have undefined behavior even if the translation produced a string with the wrong number of "%s"s. */ { diff --git a/libdap4/d4odom.c b/libdap4/d4odom.c index 1b7f650810..ed7d4639fd 100644 --- a/libdap4/d4odom.c +++ b/libdap4/d4odom.c @@ -78,7 +78,7 @@ d4odom_print(D4odometer* odom) if(odom->rank == 0) { strlcat(line,"[]",sizeof(line)); } else for(i=0;irank;i++) { - sprintf(tmp,"[%lu/%lu:%lu:%lu]", + snprintf(tmp,sizeof(tmp),"[%lu/%lu:%lu:%lu]", (size_t)odom->index[i], (size_t)odom->start[i], (size_t)odom->stride[i], diff --git a/libdispatch/dparallel.c b/libdispatch/dparallel.c index 6a9aba720c..3968c6fd0c 100644 --- a/libdispatch/dparallel.c +++ b/libdispatch/dparallel.c @@ -76,7 +76,7 @@ parallel I/O. MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE); if ((res = nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid))) ERR; if (nc_def_dim(ncid, "d1", DIMSIZE, dimids)) ERR; diff --git a/libdispatch/nctime.c b/libdispatch/nctime.c index e265ba1f49..aa937de54b 100644 --- a/libdispatch/nctime.c +++ b/libdispatch/nctime.c @@ -386,7 +386,7 @@ cdParseRelunits(cdCalenType timetype, char* relunits, cdUnitTime* unit, cdCompTi else{ cdTrim(basetime_1,CD_MAX_CHARTIME); cdTrim(basetime_2,CD_MAX_CHARTIME); - sprintf(basetime,"%s %s",basetime_1,basetime_2); + snprintf(basetime,sizeof(basetime),"%s %s",basetime_1,basetime_2); } } diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 0c2f5b5234..6e7154493b 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -482,7 +482,7 @@ create_phony_dims(NC_GRP_INFO_T *grp, hid_t hdf_datasetid, NC_VAR_INFO_T *var) if (!match) { char phony_dim_name[NC_MAX_NAME + 1]; - sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid); + snprintf(phony_dim_name, sizeof(phony_dim_name), "phony_dim_%d", grp->nc4_info->next_dimid); LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name)); /* Add phony dim to metadata list. */ diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index a84abd0223..89cc5f973a 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -702,13 +702,13 @@ write_quantize_att(NC_VAR_INFO_T *var) switch (var->quantize_mode) { case NC_QUANTIZE_BITGROOM: - sprintf(att_name, "%s", NC_QUANTIZE_BITGROOM_ATT_NAME); + snprintf(att_name, sizeof(att_name), "%s", NC_QUANTIZE_BITGROOM_ATT_NAME); break; case NC_QUANTIZE_GRANULARBR: - sprintf(att_name, "%s", NC_QUANTIZE_GRANULARBR_ATT_NAME); + snprintf(att_name, sizeof(att_name), "%s", NC_QUANTIZE_GRANULARBR_ATT_NAME); break; case NC_QUANTIZE_BITROUND: - sprintf(att_name, "%s", NC_QUANTIZE_BITROUND_ATT_NAME); + snprintf(att_name, sizeof(att_name), "%s", NC_QUANTIZE_BITROUND_ATT_NAME); break; default: return NC_EINVAL; @@ -1826,7 +1826,7 @@ nc4_create_dim_wo_var(NC_DIM_INFO_T *dim) /* Indicate that this is a scale. Also indicate that not * be shown to the user as a variable. It is hidden. It is * a DIM WITHOUT A VARIABLE! */ - sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len); + snprintf(dimscale_wo_var, sizeof(dimscale_wo_var), "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len); if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0) BAIL(NC_EHDFERR); @@ -2257,7 +2257,7 @@ nc4_rec_match_dimscales(NC_GRP_INFO_T *grp) if (match < 0) { char phony_dim_name[NC_MAX_NAME + 1]; - sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid); + snprintf(phony_dim_name, sizeof(phony_dim_name), "phony_dim_%d", grp->nc4_info->next_dimid); LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name)); if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim))) { diff --git a/libsrc/ffio.c b/libsrc/ffio.c index 80bc6cb930..a59246e9a8 100644 --- a/libsrc/ffio.c +++ b/libsrc/ffio.c @@ -604,7 +604,7 @@ ncio_ffio_assign(const char *filename) { errno=E2BIG; return (const char *) NULL; } - (void) sprintf(buffer,"-F %s %s", envstr,xtra_assign); + (void) snprintf(buffer,sizeof(buffer),"-F %s %s", envstr,xtra_assign); #ifdef __crayx1 ASNFILE(filename, buffer, &istat, strlen(filename)+1, strlen(buffer)+1); #else diff --git a/libsrc/pstdint.h b/libsrc/pstdint.h index c0b4addb34..98936cc334 100644 --- a/libsrc/pstdint.h +++ b/libsrc/pstdint.h @@ -763,27 +763,27 @@ int main () { uintmax_t umax = UINTMAX_C(0); char str0[256], str1[256]; - sprintf (str0, "%d %x\n", 0, ~0); + snprintf (str0, sizeof(str0), "%d %x\n", 0, ~0); - sprintf (str1, "%d %x\n", i8, ~0); + snprintf (str1, sizeof(str1), "%d %x\n", i8, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with i8 : %s\n", str1); - sprintf (str1, "%u %x\n", u8, ~0); + snprintf (str1, sizeof(str1), "%u %x\n", u8, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with u8 : %s\n", str1); - sprintf (str1, "%d %x\n", i16, ~0); + snprintf (str1, sizeof(str1), "%d %x\n", i16, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with i16 : %s\n", str1); - sprintf (str1, "%u %x\n", u16, ~0); + snprintf (str1, sizeof(str1), "%u %x\n", u16, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with u16 : %s\n", str1); - sprintf (str1, "%" PRINTF_INT32_MODIFIER "d %x\n", i32, ~0); + snprintf (str1, sizeof(str1), "%" PRINTF_INT32_MODIFIER "d %x\n", i32, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with i32 : %s\n", str1); - sprintf (str1, "%" PRINTF_INT32_MODIFIER "u %x\n", u32, ~0); + snprintf (str1, sizeof(str1), "%" PRINTF_INT32_MODIFIER "u %x\n", u32, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with u32 : %s\n", str1); #ifdef INT64_MAX - sprintf (str1, "%" PRINTF_INT64_MODIFIER "d %x\n", i64, ~0); + snprintf (str1, sizeof(str1), "%" PRINTF_INT64_MODIFIER "d %x\n", i64, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with i64 : %s\n", str1); #endif - sprintf (str1, "%" PRINTF_INTMAX_MODIFIER "d %x\n", imax, ~0); + snprintf (str1, sizeof(str1), "%" PRINTF_INTMAX_MODIFIER "d %x\n", imax, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with imax : %s\n", str1); - sprintf (str1, "%" PRINTF_INTMAX_MODIFIER "u %x\n", umax, ~0); + snprintf (str1, sizeof(str1), "%" PRINTF_INTMAX_MODIFIER "u %x\n", umax, ~0); if (0 != strcmp (str0, str1)) printf ("Something wrong with umax : %s\n", str1); TESTUMAX(8); diff --git a/libsrc4/nc4internal.c b/libsrc4/nc4internal.c index 2c6e64d1b3..fc5f62aed6 100644 --- a/libsrc4/nc4internal.c +++ b/libsrc4/nc4internal.c @@ -1742,7 +1742,7 @@ nc4_init_logging(void) } /* Create a filename with the rank in it. */ - sprintf(log_filename, "nc4_log_%d.log", my_rank); + snprintf(log_filename, sizeof(log_filename), "nc4_log_%d.log", my_rank); /* Open a file for this rank to log messages. */ if (!(LOG_FILE = fopen(log_filename, "w"))) @@ -1875,7 +1875,7 @@ rec_print_metadata(NC_GRP_INFO_T *grp, int tab_count) strcpy(dims_string, ""); for (d = 0; d < var->ndims; d++) { - sprintf(temp_string, " %d", var->dimids[d]); + snprintf(temp_string, sizeof(temp_string), " %d", var->dimids[d]); strcat(dims_string, temp_string); } } diff --git a/nc_perf/bm_file.c b/nc_perf/bm_file.c index ee04cfdd7a..925c14e009 100644 --- a/nc_perf/bm_file.c +++ b/nc_perf/bm_file.c @@ -1075,8 +1075,8 @@ main(int argc, char **argv) #endif /* Create a copy of file_out. This will defeat any buffering * that may exist from the fact that we just wrote file_out. */ - sprintf(file_out_2, "tst_copy_%s", file_out); - sprintf(cmd, "cp %s %s\n", file_out, file_out_2); + snprintf(file_out_2, sizeof(file_out_2), "tst_copy_%s", file_out); + snprintf(cmd, sizeof(cmd), "cp %s %s\n", file_out, file_out_2); system(cmd); if ((ret = cmp_file(file_in, file_out_2, &meta_read2_us, &data_read2_us, diff --git a/nc_perf/bm_many_atts.c b/nc_perf/bm_many_atts.c index 7cccf6563c..07334110c9 100644 --- a/nc_perf/bm_many_atts.c +++ b/nc_perf/bm_many_atts.c @@ -58,12 +58,12 @@ int main(int argc, char **argv) ERR; for(g = 1; g < numgrp + 1; g++) { - sprintf(gname, "group%d", g); + snprintf(gname, sizeof(gname), "group%d", g); if (nc_def_grp(ncid, gname, &grp)) ERR; natts = g < numgrp ? NC_MAX_ATTRS : aleft; /* leftovers on last time through */ for(an = 1; an < natts + 1; an++) { char aname[20]; - sprintf(aname, "attribute%d", a); + snprintf(aname, sizeof(aname), "attribute%d", a); if (nc_put_att_int(grp, NC_GLOBAL, aname, NC_INT, 1, data)) ERR; if(a%100 == 0) { /* only print every 100th attribute name */ if (gettimeofday(&end_time, NULL)) ERR; diff --git a/nc_perf/bm_many_objs.c b/nc_perf/bm_many_objs.c index defc65a143..43e4f49da9 100644 --- a/nc_perf/bm_many_objs.c +++ b/nc_perf/bm_many_objs.c @@ -52,7 +52,7 @@ int main(int argc, char **argv) /* create N groups, printing time after every 1000 */ numgrp = nitem; for(g = 1; g < numgrp + 1; g++) { - sprintf(gname, "group%d", g); + snprintf(gname, sizeof(gname), "group%d", g); if (nc_def_grp(ncid, gname, &grp)) ERR; if (nc_def_var(grp, "var", NC_INT, 0, NULL, &var)) ERR; if(nc_enddef (grp)) ERR; @@ -80,13 +80,13 @@ int main(int argc, char **argv) ERR; for(g = 1; g < numgrp + 1; g++) { - sprintf(gname, "group%d", g); + snprintf(gname, sizeof(gname), "group%d", g); if (nc_def_grp(ncid, gname, &grp)) ERR; nvars = g < numgrp ? NC_MAX_VARS : vleft; /* leftovers on last time through */ for(vn = 1; vn < nvars + 1; vn++) { int var; char vname[20]; - sprintf(vname, "variable%d", v); + snprintf(vname, sizeof(vname), "variable%d", v); if(nc_def_var(grp, vname, NC_INT, 0, NULL, &var)) ERR; if(nc_put_var(grp, var, data)) ERR; if(v%1000 == 0) { /* only print every 1000th variable name */ diff --git a/nc_perf/tst_attsperf.c b/nc_perf/tst_attsperf.c index 26a2b48440..d9911ea397 100644 --- a/nc_perf/tst_attsperf.c +++ b/nc_perf/tst_attsperf.c @@ -48,7 +48,7 @@ add_attributes(int ncid, int varid, size_t num_atts, size_t att_len) /* Write a bunch of attributes. */ for (a = 0; a < num_atts; a++) { - sprintf(att_name, "%s_varid_%d_att_%d", TEST, varid, a); + snprintf(att_name, sizeof(att_name), "%s_varid_%d_att_%d", TEST, varid, a); if (nc_put_att_double(ncid, varid, att_name, NC_DOUBLE, att_len, att_data)) ERR; } @@ -73,7 +73,7 @@ buildfile(size_t num_vars, size_t num_atts, size_t att_len, char *file_name) for (v = 0; v < num_vars; v++) { char var_name[NC_MAX_NAME + 1]; - sprintf(var_name, "%s_var_%d", TEST, v); + snprintf(var_name, sizeof(var_name), "%s_var_%d", TEST, v); if (nc_def_var(ncid, var_name, NC_INT, NDIMS, dimids, &varid)) ERR; if (add_attributes(ncid, v, num_atts, att_len)) ERR; } @@ -200,7 +200,7 @@ main(int argc, char **argv) long long hdf5_open_time = 0; /* Determine file name. */ - sprintf(file_name, "%s_%d_%d.nc", TEST, num_vars, r); + snprintf(file_name, sizeof(file_name), "%s_%d_%d.nc", TEST, num_vars, r); if (buildfile(num_vars, num_atts, ATT_LEN, file_name)) ERR; if (readfile(file_name, &nc4_open_time, do_inq, num_vars)) ERR; @@ -242,7 +242,7 @@ main(int argc, char **argv) long long hdf5_open_time; /* Determine file name. */ - sprintf(file_name, "%s_%d_%d_%d.nc", TEST, num_vars, s, r); + snprintf(file_name, sizeof(file_name), "%s_%d_%d_%d.nc", TEST, num_vars, s, r); if (buildfile(num_vars, num_atts, ATT_LEN, file_name)) ERR; if (readfile(file_name, &nc4_open_time, do_inq, num_vars)) ERR; diff --git a/nc_perf/tst_bm_rando.c b/nc_perf/tst_bm_rando.c index 05dfd216b6..825c49af20 100644 --- a/nc_perf/tst_bm_rando.c +++ b/nc_perf/tst_bm_rando.c @@ -74,7 +74,7 @@ main(int argc, char **argv) if (nc_def_dim(ncid, DIM2_NAME, DIM2_LEN, &dimid[2])) ERR; for (v = 0; v < NUM_VAR; v++) { - sprintf(name, "var_%d", v); + snprintf(name, sizeof(name), "var_%d", v); if (nc_def_var(ncid, name, NC_FLOAT, NDIM3, dimid, &varid[v])) ERR; if (nc_def_var_chunking(ncid, v, NC_CHUNKED, chunksize)) ERR; } @@ -103,8 +103,8 @@ main(int argc, char **argv) { /* Create a copy of file_out. This will defeat any OS * buffering. */ - sprintf(file_2, "tst_copy_%d_%s", c, FILE_NAME); - sprintf(cmd, "cp %s %s\n", FILE_NAME, file_2); + snprintf(file_2, sizeof(file_2), "tst_copy_%d_%s", c, FILE_NAME); + snprintf(cmd, sizeof(cmd), "cp %s %s\n", FILE_NAME, file_2); system(cmd); } else diff --git a/nc_perf/tst_chunks3.c b/nc_perf/tst_chunks3.c index 701a2c4bae..1cc9168462 100644 --- a/nc_perf/tst_chunks3.c +++ b/nc_perf/tst_chunks3.c @@ -307,7 +307,7 @@ main(int argc, char *argv[]) { count[1] = dims[1]; count[2] = dims[2]; - sprintf(time_mess," contiguous write %3d %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," contiguous write %3d %3ld %3ld", 1, dims[1], dims[2]); TIMING_START ; for(i = 0; i < dims[0]; i++) { @@ -319,7 +319,7 @@ main(int argc, char *argv[]) { printf("\n"); contig_time = TMsec; - sprintf(time_mess," chunked write %3d %3ld %3ld %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," chunked write %3d %3ld %3ld %3ld %3ld %3ld", 1, dims[1], dims[2], chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[0]; i++) { @@ -335,7 +335,7 @@ main(int argc, char *argv[]) { else printf(" %5.2g x slower\n", 1.0/ratio); - sprintf(time_mess," compressed write %3d %3ld %3ld %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," compressed write %3d %3ld %3ld %3ld %3ld %3ld", 1, dims[1], dims[2], chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[0]; i++) { @@ -360,7 +360,7 @@ main(int argc, char *argv[]) { count[1] = 1; count[2] = dims[2]; - sprintf(time_mess," contiguous write %3ld %3d %3ld", + snprintf(time_mess, sizeof(time_mess)," contiguous write %3ld %3d %3ld", dims[0], 1, dims[2]); TIMING_START ; for(i = 0; i < dims[1]; i++) { @@ -372,7 +372,7 @@ main(int argc, char *argv[]) { printf("\n"); contig_time = TMsec; - sprintf(time_mess," chunked write %3ld %3d %3ld %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," chunked write %3ld %3d %3ld %3ld %3ld %3ld", dims[0], 1, dims[2], chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[1]; i++) { @@ -388,7 +388,7 @@ main(int argc, char *argv[]) { else printf(" %5.2g x slower\n", 1.0/ratio); - sprintf(time_mess," compressed write %3ld %3d %3ld %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," compressed write %3ld %3d %3ld %3ld %3ld %3ld", dims[0], 1, dims[2], chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[1]; i++) { @@ -413,7 +413,7 @@ main(int argc, char *argv[]) { count[1] = dims[1]; count[2] = 1; - sprintf(time_mess," contiguous write %3ld %3ld %3d", + snprintf(time_mess, sizeof(time_mess)," contiguous write %3ld %3ld %3d", dims[0], dims[1], 1); TIMING_START ; for(i = 0; i < dims[2]; i++) { @@ -425,7 +425,7 @@ main(int argc, char *argv[]) { printf("\n"); contig_time = TMsec; - sprintf(time_mess," chunked write %3ld %3ld %3d %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," chunked write %3ld %3ld %3d %3ld %3ld %3ld", dims[0], dims[1], 1, chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[2]; i++) { @@ -441,7 +441,7 @@ main(int argc, char *argv[]) { else printf(" %5.2g x slower\n", 1.0/ratio); - sprintf(time_mess," compressed write %3ld %3ld %3d %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," compressed write %3ld %3ld %3d %3ld %3ld %3ld", dims[0], dims[1], 1, chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[2]; i++) { @@ -466,7 +466,7 @@ main(int argc, char *argv[]) { count[1] = dims[1]; count[2] = dims[2]; - sprintf(time_mess," contiguous read %3d %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," contiguous read %3d %3ld %3ld", 1, dims[1], dims[2]); TIMING_START ; for(i = 0; i < dims[0]; i++) { @@ -478,7 +478,7 @@ main(int argc, char *argv[]) { printf("\n"); contig_time = TMsec; - sprintf(time_mess," chunked read %3d %3ld %3ld %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," chunked read %3d %3ld %3ld %3ld %3ld %3ld", 1, dims[1], dims[2] , chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[0]; i++) { @@ -494,7 +494,7 @@ main(int argc, char *argv[]) { else printf(" %5.2g x slower\n", 1.0/ratio); - sprintf(time_mess," compressed read %3d %3ld %3ld %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," compressed read %3d %3ld %3ld %3ld %3ld %3ld", 1, dims[1], dims[2] , chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[0]; i++) { @@ -519,7 +519,7 @@ main(int argc, char *argv[]) { count[1] = 1; count[2] = dims[2]; - sprintf(time_mess," contiguous read %3ld %3d %3ld", + snprintf(time_mess, sizeof(time_mess)," contiguous read %3ld %3d %3ld", dims[0], 1, dims[2]); TIMING_START ; for(i = 0; i < dims[1]; i++) { @@ -531,7 +531,7 @@ main(int argc, char *argv[]) { printf("\n"); contig_time = TMsec; - sprintf(time_mess," chunked read %3ld %3d %3ld %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," chunked read %3ld %3d %3ld %3ld %3ld %3ld", dims[0], 1, dims[2], chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[1]; i++) { @@ -547,7 +547,7 @@ main(int argc, char *argv[]) { else printf(" %5.2g x slower\n", 1.0/ratio); - sprintf(time_mess," compressed read %3ld %3d %3ld %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," compressed read %3ld %3d %3ld %3ld %3ld %3ld", dims[0], 1, dims[2], chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[1]; i++) { @@ -572,7 +572,7 @@ main(int argc, char *argv[]) { count[1] = dims[1]; count[2] = 1; - sprintf(time_mess," contiguous read %3ld %3ld %3d", + snprintf(time_mess, sizeof(time_mess)," contiguous read %3ld %3ld %3d", dims[0], dims[1], 1); TIMING_START ; for(i = 0; i < dims[2]; i++) { @@ -584,7 +584,7 @@ main(int argc, char *argv[]) { printf("\n"); contig_time = TMsec; - sprintf(time_mess," chunked read %3ld %3ld %3d %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," chunked read %3ld %3ld %3d %3ld %3ld %3ld", dims[0], dims[1], 1, chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[2]; i++) { @@ -600,7 +600,7 @@ main(int argc, char *argv[]) { else printf(" %5.2g x slower\n", 1.0/ratio); - sprintf(time_mess," compressed read %3ld %3ld %3d %3ld %3ld %3ld", + snprintf(time_mess, sizeof(time_mess)," compressed read %3ld %3ld %3d %3ld %3ld %3ld", dims[0], dims[1], 1, chunks[0], chunks[1], chunks[2]); TIMING_START ; for(i = 0; i < dims[2]; i++) { diff --git a/nc_perf/tst_compress.c b/nc_perf/tst_compress.c index 1495ff4848..bd3a816cc8 100644 --- a/nc_perf/tst_compress.c +++ b/nc_perf/tst_compress.c @@ -282,7 +282,7 @@ write_meta(int ncid, int *data_varid, int s, int f, int nsd, int deflate, int u, { char data_var_name[NC_MAX_NAME + 1]; - sprintf(data_var_name, "var_%d", dv); + snprintf(data_var_name, sizeof(data_var_name), "var_%d", dv); if (nc_redef(ncid)) ERR; if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; @@ -567,7 +567,7 @@ main(int argc, char **argv) /* Use the same filename every time, so we don't * create many large files, just one. ;-) */ - sprintf(file_name, "%s.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s.nc", TEST_NAME); /* Remove the last file. Ignore errors. */ remove(file_name); diff --git a/nc_perf/tst_compress_par.c b/nc_perf/tst_compress_par.c index 39b769c960..c38b494b12 100644 --- a/nc_perf/tst_compress_par.c +++ b/nc_perf/tst_compress_par.c @@ -290,7 +290,7 @@ write_meta(int ncid, int *data_varid, int s, int f, int nsd, int deflate, int u, { char data_var_name[NC_MAX_NAME + 1]; - sprintf(data_var_name, "var_%d", dv); + snprintf(data_var_name, sizeof(data_var_name), "var_%d", dv); if (nc_redef(ncid)) ERR; if (nc_def_var(ncid, data_var_name, NC_FLOAT, NDIM4, dimid_data, &data_varid[dv])) ERR; @@ -641,7 +641,7 @@ main(int argc, char **argv) /* Use the same filename every time, so we don't * create many large files, just one. ;-) */ - sprintf(file_name, "%s.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s.nc", TEST_NAME); /* nc_set_log_level(3); */ /* Create a parallel netcdf-4 file. */ diff --git a/nc_perf/tst_create_files.c b/nc_perf/tst_create_files.c index c3fdd87460..753b3979ac 100644 --- a/nc_perf/tst_create_files.c +++ b/nc_perf/tst_create_files.c @@ -50,7 +50,7 @@ main(int argc, char **argv) /* User TEMP_LARGE as the directory. */ if (strlen(TEMP_LARGE) + strlen(LARGE_FILE) > NC_MAX_NAME * 2) ERR; - sprintf(file_name, "%s/%s", TEMP_LARGE, LARGE_FILE); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, LARGE_FILE); /* Create file with 3 dims, one variable. */ if (nc_create(file_name, NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; @@ -171,11 +171,11 @@ main(int argc, char **argv) * dimensions. */ for (ndims = 1; ndims <= MAX_DIMS; ndims++) { - sprintf(file_name, "tst_%s2_%dD.nc", type_name[t], ndims); + snprintf(file_name, sizeof(file_name), "tst_%s2_%dD.nc", type_name[t], ndims); if (nc_create(file_name, 0, &ncid)) ERR; for (len = pow(TOTAL_SIZE, (float)1/ndims), d = 0; d < ndims; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, len, &dimids[d])) ERR; } if (nc_def_var(ncid, VAR_NAME, typeid[t], ndims, dimids, &varid)) ERR; @@ -284,11 +284,11 @@ main(int argc, char **argv) * dimensions. */ for (ndims = 1; ndims <= MAX_DIMS; ndims++) { - sprintf(file_name, "tst_%s2_%dD.nc", type_name[t], ndims); + snprintf(file_name, sizeof(file_name), "tst_%s2_%dD.nc", type_name[t], ndims); if (nc_create(file_name, 0, &ncid)) ERR; for (len = pow(TOTAL_SIZE, (float)1/ndims), d = 0; d < ndims; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, len, &dimids[d])) ERR; } if (nc_def_var(ncid, SIMPLE_VAR_NAME, typeid[t], ndims, dimids, &varid)) ERR; diff --git a/nc_perf/tst_files2.c b/nc_perf/tst_files2.c index ca9c8211ac..ccc8580edc 100644 --- a/nc_perf/tst_files2.c +++ b/nc_perf/tst_files2.c @@ -41,7 +41,7 @@ get_mem_used1(int *mem_used) /* Run the ps command for this process, putting output (one number) * into file TMP_FILE_NAME. */ - sprintf(cmd, "ps -o size= %d > %s", getpid(), TMP_FILE_NAME); + snprintf(cmd, sizeof(cmd), "ps -o size= %d > %s", getpid(), TMP_FILE_NAME); system(cmd); /* Read the results and delete temp file. */ @@ -116,7 +116,7 @@ create_sample_file(char *file_name, int ndims, int *dim_len, /* Create the dimensions. */ for (d = 0; d < ndims; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR_RET; } @@ -124,7 +124,7 @@ create_sample_file(char *file_name, int ndims, int *dim_len, if (!(varids = malloc(num_vars * sizeof(int)))) ERR_RET; for (i = 0; i < num_vars; i++) { - sprintf(varname, "a_%d", i); + snprintf(varname, sizeof(varname), "a_%d", i); if (nc_def_var(ncid, varname, NC_FLOAT, ndims, dimids, &varids[i])) ERR_RET; } @@ -218,7 +218,7 @@ main(int argc, char **argv) for (f = 0; f < num_files[t]; f++) { /* Set up filename. */ - sprintf(file_name[t], "tst_files2_%d_%d.nc", t, f); + snprintf(file_name[t], sizeof(file_name[t]), "tst_files2_%d_%d.nc", t, f); if (create_sample_file(file_name[t], ndims[t], dim_len[t], num_vars[t], mode[t], num_recs[t])) ERR; @@ -273,20 +273,20 @@ main(int argc, char **argv) /* Prepare the dimensions string. */ if (ndims[t] == MAX_DIMS) - sprintf(dimstr, "%dx%dx%dx%d", dim_len[t][0], dim_len[t][1], + snprintf(dimstr, sizeof(dimstr), "%dx%dx%dx%d", dim_len[t][0], dim_len[t][1], dim_len[t][2], dim_len[t][3]); else - sprintf(dimstr, "%dx%dx%d", dim_len[t][0], dim_len[t][1], + snprintf(dimstr, sizeof(dimstr), "%dx%dx%d", dim_len[t][0], dim_len[t][1], dim_len[t][2]); /* Prepare the chunksize string. */ if (storage == NC_CHUNKED) { if (ndims[t] == MAX_DIMS) - sprintf(chunkstr, "%dx%dx%dx%d", (int)chunksize[0], (int)chunksize[1], + snprintf(chunkstr, sizeof(chunkstr), "%dx%dx%dx%d", (int)chunksize[0], (int)chunksize[1], (int)chunksize[2], (int)chunksize[3]); else - sprintf(chunkstr, "%dx%dx%d", (int)chunksize[0], (int)chunksize[1], + snprintf(chunkstr, sizeof(chunkstr), "%dx%dx%d", (int)chunksize[0], (int)chunksize[1], (int)chunksize[2]); } else diff --git a/nc_perf/tst_h_many_atts.c b/nc_perf/tst_h_many_atts.c index f484c43103..f5cd1f4129 100644 --- a/nc_perf/tst_h_many_atts.c +++ b/nc_perf/tst_h_many_atts.c @@ -58,7 +58,7 @@ main() if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0) ERR; for (i = 0; i < NUM_ATTS; i++) { - sprintf(name, "att_%d", i); + snprintf(name, sizeof(name), "att_%d", i); if ((attid1 = H5Acreate2(grpid, name, H5T_NATIVE_INT, spaceid, H5P_DEFAULT, H5P_DEFAULT)) < 0) ERR; if (H5Awrite(attid1, H5T_NATIVE_INT, &one) < 0) ERR; diff --git a/nc_perf/tst_wrf_reads.c b/nc_perf/tst_wrf_reads.c index e38805fb28..34d998f035 100644 --- a/nc_perf/tst_wrf_reads.c +++ b/nc_perf/tst_wrf_reads.c @@ -26190,7 +26190,7 @@ main(int argc, char **argv) char file_name[NC_MAX_NAME + 1]; nc_set_log_level(4); - sprintf(file_name, "%s_wrf_chem_%d.nc", FILE_NAME, i); + snprintf(file_name, sizeof(file_name), "%s_wrf_chem_%d.nc", FILE_NAME, i); run_test(2, file_name, cmode[c], &open_time, &create_time, &close_time); remove(file_name); } @@ -26214,7 +26214,7 @@ main(int argc, char **argv) /* char file_name[NC_MAX_NAME + 1]; */ /* nc_set_log_level(4); */ - /* sprintf(file_name, "%s_%d.nc", FILE_NAME, i); */ + /* snprintf(file_name, sizeof(file_name), "%s_%d.nc", FILE_NAME, i); */ /* run_test(0, file_name, cmode[c], &open_time, &create_time, &close_time); */ /* remove(file_name); */ /* } */ @@ -26237,7 +26237,7 @@ main(int argc, char **argv) /* char file_name[NC_MAX_NAME + 1]; */ /* nc_set_log_level(4); */ - /* sprintf(file_name, "%s_%d.nc", FILE_NAME, i); */ + /* snprintf(file_name, sizeof(file_name), "%s_%d.nc", FILE_NAME, i); */ /* run_test(1, file_name, cmode[c], &open_time, &create_time, &close_time); */ /* remove(file_name); */ /* } */ diff --git a/nc_test/large_files.c b/nc_test/large_files.c index 114fa3d2fd..ec4f1f0884 100644 --- a/nc_test/large_files.c +++ b/nc_test/large_files.c @@ -72,7 +72,7 @@ main(int argc, char **argv) { printf("\n*** Testing large files, slowly.\n"); - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE_NAME); printf("*** Creating large file %s...", file_name); /* enter define mode */ diff --git a/nc_test/quick_large_files.c b/nc_test/quick_large_files.c index 756a6faa62..7d2e385dee 100644 --- a/nc_test/quick_large_files.c +++ b/nc_test/quick_large_files.c @@ -65,7 +65,7 @@ main(int argc, char **argv) } /* Create a netCDF 64-bit offset format file. Write a value. */ - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE_NAME); printf("*** Creating %s for 64-bit offset large file test...", file_name); { if ((res = nc_create(file_name, cflag|NC_64BIT_OFFSET, &ncid))) diff --git a/nc_test/testnc3perf.c b/nc_test/testnc3perf.c index 3f97c3f290..b4f0a8a872 100644 --- a/nc_test/testnc3perf.c +++ b/nc_test/testnc3perf.c @@ -298,7 +298,7 @@ test_slabs(ncid, sizes) edge[idim] = dims[idim].size; } - sprintf(time_mess,"ncvarput %ldx%ldx%ldx%ld", + snprintf(time_mess, sizeof(time_mess),"ncvarput %ldx%ldx%ldx%ld", edge[0], edge[1], edge[2], edge[3]); TIMING_START ; @@ -321,7 +321,7 @@ test_slabs(ncid, sizes) point[idim] = corner[idim]; } - sprintf(time_mess,"ncvarget %ldx%ldx%ldx%ld" + snprintf(time_mess, sizeof(time_mess),"ncvarget %ldx%ldx%ldx%ld" ,edge[0],edge[1],edge[2],edge[3]); TIMING_START ; @@ -343,7 +343,7 @@ test_slabs(ncid, sizes) corner[idim] = 0; /* get vector along dimension idim */ edge[idim] = dims[idim].size; - sprintf(time_mess,"ncvarget %ldx%ldx%ldx%ld" + snprintf(time_mess, sizeof(time_mess),"ncvarget %ldx%ldx%ldx%ld" ,edge[0],edge[1],edge[2],edge[3]); TIMING_START ; @@ -372,7 +372,7 @@ test_slabs(ncid, sizes) edge[idim] = dims[idim].size; edge[jdim] = dims[jdim].size; - sprintf(time_mess,"ncvarget %ldx%ldx%ldx%ld" + snprintf(time_mess, sizeof(time_mess),"ncvarget %ldx%ldx%ldx%ld" ,edge[0],edge[1],edge[2],edge[3]); TIMING_START ; @@ -411,7 +411,7 @@ test_slabs(ncid, sizes) edge[jdim] = dims[jdim].size; edge[kdim] = dims[kdim].size; - sprintf(time_mess,"ncvarget %ldx%ldx%ldx%ld" + snprintf(time_mess, sizeof(time_mess),"ncvarget %ldx%ldx%ldx%ld" ,edge[0],edge[1],edge[2],edge[3]); TIMING_START ; @@ -447,7 +447,7 @@ test_slabs(ncid, sizes) edge[idim] = dims[idim].size; } - sprintf(time_mess,"ncvarget %ldx%ldx%ldx%ld" + snprintf(time_mess, sizeof(time_mess),"ncvarget %ldx%ldx%ldx%ld" ,edge[0],edge[1],edge[2],edge[3]); TIMING_START ; diff --git a/nc_test/tst_addvar.c b/nc_test/tst_addvar.c index bde8bba25a..01d6100b3d 100644 --- a/nc_test/tst_addvar.c +++ b/nc_test/tst_addvar.c @@ -74,7 +74,7 @@ int main(int argc, char** argv) { for (i=0; i<3; i++) { char varname[32]; - sprintf(varname, "fixed_var_%d",2*i+1); + snprintf(varname, sizeof(varname), "fixed_var_%d",2*i+1); for (j=0; j<5; j++) { if (get_buf[i][j] != old_buf[i][j]) { printf("Error in %s line %d: expecting %s[%d]=%d but got %d\n", diff --git a/nc_test/tst_big_rvar.c b/nc_test/tst_big_rvar.c index dd885120a3..5df0079f73 100644 --- a/nc_test/tst_big_rvar.c +++ b/nc_test/tst_big_rvar.c @@ -112,7 +112,7 @@ main(int argc, char **argv) { char testfile[NC_MAX_NAME + 1]; printf("\n*** Testing files with multidimensional variable with more than 2**32 values\n"); - sprintf(testfile, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(testfile, sizeof(testfile), "%s/%s", TEMP_LARGE, FILE_NAME); for (i = NC_FORMAT_CLASSIC; i <= NUM_FORMATS; i++) { printf("*** testing format %d file with record variable with > 2**32 values...", i); diff --git a/nc_test/tst_big_var.c b/nc_test/tst_big_var.c index 98dc8dfc68..d5734ab07c 100644 --- a/nc_test/tst_big_var.c +++ b/nc_test/tst_big_var.c @@ -70,7 +70,7 @@ main(int argc, char **argv) { char testfile[NC_MAX_NAME + 1]; printf("\n*** Testing files with one very big variable.\n"); - sprintf(testfile, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(testfile, sizeof(testfile), "%s/%s", TEMP_LARGE, FILE_NAME); for (i = NC_FORMAT_CLASSIC; i <= NUM_FORMATS; i++) { printf("*** testing format %d file with byte variable with > 2**32 values...", i); diff --git a/nc_test/tst_big_var2.c b/nc_test/tst_big_var2.c index f3c2cf3160..78c36bbfba 100644 --- a/nc_test/tst_big_var2.c +++ b/nc_test/tst_big_var2.c @@ -121,7 +121,7 @@ main(int argc, char **argv) { char testfile[NC_MAX_NAME + 1]; printf("\n*** Testing multidimensional variable with more than 2**32 values\n"); - sprintf(testfile, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(testfile, sizeof(testfile), "%s/%s", TEMP_LARGE, FILE_NAME); for (i = NC_FORMAT_CLASSIC; i <= NUM_FORMATS; i++) { printf("*** testing format %d file with byte variable with > 2**32 values...", i); diff --git a/nc_test/tst_big_var6.c b/nc_test/tst_big_var6.c index b237a50317..ad85ede919 100644 --- a/nc_test/tst_big_var6.c +++ b/nc_test/tst_big_var6.c @@ -102,7 +102,7 @@ main(int argc, char **argv) { char testfile[NC_MAX_NAME + 1]; printf("\n*** Testing multidimensional variable with more than 2**32 values\n"); - sprintf(testfile, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(testfile, sizeof(testfile), "%s/%s", TEMP_LARGE, FILE_NAME); for (i = NC_FORMAT_CLASSIC; i <= NUM_FORMATS; i++) { printf("*** testing format %d file with short variable with > 2**32 values...", i); diff --git a/nc_test/tst_cdf5format.c b/nc_test/tst_cdf5format.c index f5162ab782..68ebf9306a 100644 --- a/nc_test/tst_cdf5format.c +++ b/nc_test/tst_cdf5format.c @@ -35,12 +35,12 @@ write2(int ncid, int parallel) { if (i % 2) { - sprintf(str, "fixed_var_%d",i); + snprintf(str, sizeof(str), "fixed_var_%d",i); if (nc_def_var(ncid, str, NC_INT, 1, &dimid[1], &varid[i])) ERR; } else { - sprintf(str, "record_var_%d",i); + snprintf(str, sizeof(str), "record_var_%d",i); if (nc_def_var(ncid, str, NC_INT, 2, dimid, &varid[i])) ERR; } } @@ -86,7 +86,7 @@ extend(int ncid) /* add attributes to make header grow */ for (i = 0; i < NVARS; i++) { - sprintf(str, "annotation_for_var_%d", i); + snprintf(str, sizeof(str), "annotation_for_var_%d", i); if (nc_put_att_text(ncid, i, "text_attr", strlen(str), str)) ERR; } if (nc_enddef(ncid)) ERR; diff --git a/nc_test/tst_diskless3.c b/nc_test/tst_diskless3.c index 2c428f33c4..31765fa958 100644 --- a/nc_test/tst_diskless3.c +++ b/nc_test/tst_diskless3.c @@ -133,7 +133,7 @@ test_two_growing_with_att(const char *testfile) { count[0] = 1; start[0] = r; - sprintf(att_name, "a_%d", data[r]); + snprintf(att_name, sizeof(att_name), "a_%d", data[r]); for (v = 0; v < NUM_VARS; v++) { if((status=nc_put_vara_text(ncid, varid[v], start, count, &data[r]))) ERRSTAT(status); diff --git a/nc_test/tst_diskless4.c b/nc_test/tst_diskless4.c index be667fe745..8519f87b95 100644 --- a/nc_test/tst_diskless4.c +++ b/nc_test/tst_diskless4.c @@ -124,7 +124,7 @@ main(int argc, char **argv) for(iv=0;iv 0) { /* system error */ const char *cp = (const char *) strerror(err); if (cp == NULL) - sprintf(unknown_str,"Unknown error code %d",err); + snprintf(unknown_str,sizeof(unknown_str),"Unknown error code %d",err); else - sprintf(unknown_str,"Error code %d (%s)",err,cp); + snprintf(unknown_str,sizeof(unknown_str),"Error code %d (%s)",err,cp); return unknown_str; } @@ -1396,7 +1396,7 @@ char* nc_err_code_name(int err) #endif #endif default: - sprintf(unknown_str,"Unknown code %d",err); + snprintf(unknown_str,sizeof(unknown_str),"Unknown code %d",err); } return unknown_str; } diff --git a/nc_test4/tst_atts2.c b/nc_test4/tst_atts2.c index eb3201511d..700e34ff65 100644 --- a/nc_test4/tst_atts2.c +++ b/nc_test4/tst_atts2.c @@ -133,7 +133,7 @@ main(int argc, char **argv) if (nc_create(FILE_NAME1, NC_NETCDF4, &ncid1)) ERR; for (a = 0; a < NUM_ATT; a++) { - sprintf(name[a], "atomic_att_type_%d", a + 1); + snprintf(name[a], sizeof(name[a]), "atomic_att_type_%d", a + 1); nc_put_att(ncid1, NC_GLOBAL, name[a], a + 1, ATT_LEN, (void *)&data); } diff --git a/nc_test4/tst_chunks.c b/nc_test4/tst_chunks.c index 1391f1eab8..280c644023 100644 --- a/nc_test4/tst_chunks.c +++ b/nc_test4/tst_chunks.c @@ -90,7 +90,7 @@ main(int argc, char **argv) if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR; for (d = 0; d < NUM_DIM; d++) { - sprintf(dim_name, "dim_%d", dim_len[d]); + snprintf(dim_name, sizeof(dim_name), "dim_%d", dim_len[d]); #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE printf("creating dim %s\n", dim_name); #endif @@ -99,7 +99,7 @@ main(int argc, char **argv) for (t = 0; t < NUM_TYPE; t++) { - sprintf(var_name, "var_%d", type_id[t]); + snprintf(var_name, sizeof(var_name), "var_%d", type_id[t]); if (nc_def_var(ncid, var_name, type_id[t], NUM_DIM, dimid, &varid[t])) ERR; if (nc_inq_var_chunking(ncid, varid[t], &contig, chunksize_in)) ERR; #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE @@ -120,7 +120,7 @@ main(int argc, char **argv) for (t = 0; t < NUM_TYPE; t++) { - sprintf(var_name, "var_%d", type_id[t]); + snprintf(var_name, sizeof(var_name), "var_%d", type_id[t]); if (nc_inq_var_chunking(ncid, varid[t], &contig, chunksize_in)) ERR; if (contig) ERR; #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE @@ -216,7 +216,7 @@ main(int argc, char **argv) /* Oh that tricky Cardinal Richelieu, he had many plans! */ for (i = 0; i < NUM_PLANS; i++) { - sprintf(plan_name, "Richelieu_sneaky_plan_%d", i); + snprintf(plan_name, sizeof(plan_name), "Richelieu_sneaky_plan_%d", i); if (nc_def_var(ncid, plan_name, i % (NC_STRING - 1) + 1, NDIMS_3, dimids, &varid[i])) ERR; if (i % 2 && nc_def_var_chunking(ncid, varid[i], 0, chunksize)) ERR; diff --git a/nc_test4/tst_chunks2.c b/nc_test4/tst_chunks2.c index 0499d6f392..3970ff243e 100644 --- a/nc_test4/tst_chunks2.c +++ b/nc_test4/tst_chunks2.c @@ -161,7 +161,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -199,7 +199,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -237,7 +237,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -278,7 +278,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -318,7 +318,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -360,7 +360,7 @@ main(int argc, char **argv) for (d = 0; d < NDIMS3; d++) { dim_len[d] = rand(); - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -399,7 +399,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -438,7 +438,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } diff --git a/nc_test4/tst_compounds.c b/nc_test4/tst_compounds.c index 731ed8bc4f..a2c6da7de8 100644 --- a/nc_test4/tst_compounds.c +++ b/nc_test4/tst_compounds.c @@ -545,7 +545,7 @@ main(int argc, char **argv) /* hr data */ hr_data_out[i].starfleet_id = i; hr_data_out[i].svc_rec = data[i]; - if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; + if (snprintf(hr_data_out[i].name, sizeof(hr_data_out[i].name), "alien_%d", i) < 0) ERR; hr_data_out[i].max_temp = 99.99f; hr_data_out[i].min_temp = -9.99f; hr_data_out[i].percent_transporter_errosion = .030303; @@ -862,7 +862,7 @@ main(int argc, char **argv) /* Create some phony data. */ for (i = 0; i < DIM1_LEN; i++) { - if (sprintf(hr_data_out[i].name, "alien_%d", i) < 0) ERR; + if (snprintf(hr_data_out[i].name, sizeof(hr_data_out[i].name), "alien_%d", i) < 0) ERR; hr_data_out[i].max_temp = 99.99f; } diff --git a/nc_test4/tst_compounds3.c b/nc_test4/tst_compounds3.c index d388658bbb..ce4a495d4f 100644 --- a/nc_test4/tst_compounds3.c +++ b/nc_test4/tst_compounds3.c @@ -173,11 +173,11 @@ main(int argc, char **argv) if (strcmp(name_in, TYPE2_NAME) || size_in != sizeof(g2_d_t)) ERR; /* This fails because it's not a fully-qualified name. */ - sprintf(full_name, "%s/%s", GROUP2_NAME, TYPE2_NAME); + snprintf(full_name, sizeof(full_name), "%s/%s", GROUP2_NAME, TYPE2_NAME); if (nc_inq_typeid(root_grp, full_name, &type2id) != NC_EINVAL) ERR; /* Check the type using it's full name. */ - sprintf(full_name, "/%s/%s", GROUP2_NAME, TYPE2_NAME); + snprintf(full_name, sizeof(full_name), "/%s/%s", GROUP2_NAME, TYPE2_NAME); /* if (nc_inq_typeid(root_grp, full_name, &type2id)) ERR; */ /* if (nc_inq_type(g2_grp, type2id, name_in, &size_in)) ERR; */ /* if (strcmp(name_in, TYPE2_NAME) || size_in != sizeof(g2_d_t)) ERR; */ diff --git a/nc_test4/tst_dims2.c b/nc_test4/tst_dims2.c index 5237d29c72..0af77d72bf 100644 --- a/nc_test4/tst_dims2.c +++ b/nc_test4/tst_dims2.c @@ -433,7 +433,7 @@ main(int argc, char **argv) { char dim_name[NC_MAX_NAME + 1]; - sprintf(dim_name, "dim_%d", i); + snprintf(dim_name, sizeof(dim_name), "dim_%d", i); if (nc_def_dim(ncid, dim_name, 1, &dimids[i])) ERR; } diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index f2ad31bedd..2a8417166a 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -329,7 +329,7 @@ main(int argc, char **argv) /* Create a bunch of files. */ for (f = 0; f < NUM_FILES; f++) { - sprintf(file_name, "tst_files2_%d.nc", f); + snprintf(file_name, sizeof(file_name), "tst_files2_%d.nc", f); if (nc_create(file_name, NC_NETCDF4, &ncid[f])) ERR; if (nc_def_dim(ncid[f], D1_NAME, TEXT_LEN + 1, &dimid)) ERR; if (nc_def_var(ncid[f], VAR_NAME, NC_CHAR, NDIMS, &dimid, &varid)) ERR; diff --git a/nc_test4/tst_grps.c b/nc_test4/tst_grps.c index 9224c96df7..30f60de15c 100644 --- a/nc_test4/tst_grps.c +++ b/nc_test4/tst_grps.c @@ -1186,11 +1186,11 @@ main(int argc, char **argv) if (nc_create(FILE_NAME, NC_CLOBBER | NC_NETCDF4, &ncid)) ERR; for (g = 0; g < PARENT_NUM_GRPS; g++) { - sprintf(grp_name, "grp_%d", g); + snprintf(grp_name, sizeof(grp_name), "grp_%d", g); if (nc_def_grp(ncid, grp_name, &g1id)) ERR; for (s = 0; s < SUB_NUM_GRPS; s++) { - sprintf(grp_name, "sub_grp_%d", s); + snprintf(grp_name, sizeof(grp_name), "sub_grp_%d", s); if (nc_def_grp(g1id, grp_name, &sub_grpid)) ERR; } } diff --git a/nc_test4/tst_interops4.c b/nc_test4/tst_interops4.c index 535a80e4fe..22fa429514 100644 --- a/nc_test4/tst_interops4.c +++ b/nc_test4/tst_interops4.c @@ -29,7 +29,7 @@ write_atts(int ncid, int varid) for (a = 0; a < NUM_ATTS; a++) { - sprintf(att_name, "att_%d", a); + snprintf(att_name, sizeof(att_name), "att_%d", a); if (nc_put_att_int(ncid, varid, att_name, NC_INT, ATT_LEN, att_data)) ERR_RET; } @@ -45,7 +45,7 @@ read_atts(int ncid, int varid) for (a = 0; a < NUM_ATTS; a++) { - sprintf(att_name, "att_%d", a); + snprintf(att_name, sizeof(att_name), "att_%d", a); if (nc_get_att_int(ncid, varid, att_name, att_data_in)) ERR_RET; for (i = 0; i < ATT_LEN; i++) @@ -67,7 +67,7 @@ write_vars(int ncid) if (nc_def_dim(ncid, DIM_NAME, VAR_LEN, NULL)) ERR; for (v = 0; v < NUM_VARS; v++) { - sprintf(var_name, "var_%d", v); + snprintf(var_name, sizeof(var_name), "var_%d", v); if (nc_def_var(ncid, var_name, NC_INT, NUM_DIMS, dimid, NULL)) ERR_RET; write_atts(ncid, v); @@ -89,7 +89,7 @@ read_vars(int ncid) { if (nc_inq_var(ncid, v, var_name_in, &xtype_in, &ndims_in, NULL, &natts_in)) ERR_RET; - sprintf(var_name, "var_%d", v); + snprintf(var_name, sizeof(var_name), "var_%d", v); if (strcmp(var_name, var_name_in) || xtype_in != NC_INT || ndims_in != NUM_DIMS || natts_in != NUM_ATTS) ERR_RET; read_atts(ncid, v); diff --git a/nc_test4/tst_interops5.c b/nc_test4/tst_interops5.c index d696eb09a7..39ec195202 100644 --- a/nc_test4/tst_interops5.c +++ b/nc_test4/tst_interops5.c @@ -99,11 +99,11 @@ main(int argc, char **argv) if (H5Sclose(ydimSpaceId) < 0) ERR; /* Create xdim scale */ - sprintf(dimNameBuf, "%s%10d", dimNameBase, nrowCur); + snprintf(dimNameBuf, sizeof(dimNameBuf), "%s%10d", dimNameBase, nrowCur); if (H5DSset_scale(xdimId, dimNameBuf) < 0) ERR; /* Create ydim scale */ - sprintf(dimNameBuf, "%s%10d", dimNameBase, ncolCur); + snprintf(dimNameBuf, sizeof(dimNameBuf), "%s%10d", dimNameBase, ncolCur); if (H5DSset_scale(ydimId, dimNameBuf) < 0) ERR; /* Attach dimension scales to the dataset */ diff --git a/nc_test4/tst_large2.c b/nc_test4/tst_large2.c index 55fea5ec9b..2c0c849c24 100644 --- a/nc_test4/tst_large2.c +++ b/nc_test4/tst_large2.c @@ -50,7 +50,7 @@ int main(int argc, char **argv) for (f = 0; f < NUM_FORMATS; f++) { printf("\t...testing with %s\n", format_name[f]); - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE_NAME); if (nc_create(file_name, this_format[f], &ncid)) ERR; if (nc_def_dim(ncid, "lat", LAT_LEN, &dimids[1])) ERR; if (nc_def_dim(ncid, "lon", LON_LEN, &dimids[2])) ERR; diff --git a/nc_test4/tst_large3.c b/nc_test4/tst_large3.c index 296f68d329..5bea1478a5 100644 --- a/nc_test4/tst_large3.c +++ b/nc_test4/tst_large3.c @@ -59,7 +59,7 @@ main(int argc, char **argv) int i; /* Create a netCDF netCDF-4/HDF5 format file, with 4 vars. */ - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE_NAME); if (nc_create(file_name, NC_NETCDF4, &ncid)) ERR; if (nc_set_fill(ncid, NC_NOFILL, NULL)) ERR; if (nc_def_dim(ncid, DIM_NAME, QTR_CLASSIC_MAX, dimids)) ERR; diff --git a/nc_test4/tst_large5.c b/nc_test4/tst_large5.c index 2b3eaf8248..20ff0447c7 100644 --- a/nc_test4/tst_large5.c +++ b/nc_test4/tst_large5.c @@ -92,7 +92,7 @@ main(int argc, char **argv) for (f = 0; f < NUM_FORMATS; f++) { printf("\t...testing with %s\n", format_name[f]); - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE_NAME); if (nc_create(file_name, this_format[f], &ncid)) ERR; if (nc_def_dim(ncid, "lat", LAT_LEN, &dimids[1])) ERR; if (nc_def_dim(ncid, "lon", LON_LEN, &dimids[2])) ERR; diff --git a/nc_test4/tst_nc4perf.c b/nc_test4/tst_nc4perf.c index af39e20943..ded16c908a 100644 --- a/nc_test4/tst_nc4perf.c +++ b/nc_test4/tst_nc4perf.c @@ -53,7 +53,7 @@ int test_pio_2d(size_t cache_size, int access_flag, MPI_Comm comm, data[j * DIMSIZE1 / mpi_size + i] = (float)mpi_rank * (j + 1); /* Get the file name. */ - sprintf(file_name, "%s/%s", TEMP_LARGE, FILENAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILENAME); /* Set the cache size. */ if (nc_get_chunk_cache(NULL, &nelems_in, &preemption_in)) ERR; @@ -114,7 +114,7 @@ int test_pio_2d(size_t cache_size, int access_flag, MPI_Comm comm, /* What was our chunking? */ if (chunk_size[0]) - sprintf(chunk_string, "%dx%d ", (int)chunk_size[0], (int)chunk_size[1]); + snprintf(chunk_string, sizeof(chunk_string), "%dx%d ", (int)chunk_size[0], (int)chunk_size[1]); else strcat(chunk_string, "contiguous"); @@ -162,7 +162,7 @@ int test_pio_4d(size_t cache_size, int access_flag, MPI_Comm comm, } /* Get the file name. */ - sprintf(file_name, "%s/%s", TEMP_LARGE, FILENAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILENAME); /* Set the cache size. */ if (nc_get_chunk_cache(NULL, &nelems_in, &preemption_in)) ERR; @@ -226,7 +226,7 @@ int test_pio_4d(size_t cache_size, int access_flag, MPI_Comm comm, /* What was our chunking? */ if (chunk_size[0]) - sprintf(chunk_string, "%dx%dx%dx%d", (int)chunk_size[0], (int)chunk_size[1], + snprintf(chunk_string, sizeof(chunk_string), "%dx%dx%dx%d", (int)chunk_size[0], (int)chunk_size[1], (int)chunk_size[2], (int)chunk_size[3]); else strcat(chunk_string, "contiguous"); diff --git a/nc_test4/tst_parallel.c b/nc_test4/tst_parallel.c index ced8b70512..25cf4a55f1 100644 --- a/nc_test4/tst_parallel.c +++ b/nc_test4/tst_parallel.c @@ -92,7 +92,7 @@ main(int argc, char **argv) /* Create a parallel netcdf-4 file. */ /*nc_set_log_level(3);*/ - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE); if ((res = nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid))) ERR; diff --git a/nc_test4/tst_parallel3.c b/nc_test4/tst_parallel3.c index 17f5e0bde5..4d711bc9be 100644 --- a/nc_test4/tst_parallel3.c +++ b/nc_test4/tst_parallel3.c @@ -83,7 +83,7 @@ int main(int argc, char **argv) facc_type_open = 0; /* Create file name. */ - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE_NAME); /* Test NetCDF4 with MPI-IO driver */ if (mpi_rank == 0) diff --git a/nc_test4/tst_parallel4.c b/nc_test4/tst_parallel4.c index b2e8df2645..7ea3c78d45 100644 --- a/nc_test4/tst_parallel4.c +++ b/nc_test4/tst_parallel4.c @@ -106,7 +106,7 @@ main(int argc, char **argv) #endif /* USE_MPE */ /* Create a parallel netcdf-4 file. */ - sprintf(file_name, "%s/%s", TEMP_LARGE, FILE_NAME); + snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE_NAME); if (nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid)) ERR; /* A global attribute holds the number of processors that created diff --git a/nc_test4/tst_parallel_zlib.c b/nc_test4/tst_parallel_zlib.c index 6e99234e88..f60a7e0679 100644 --- a/nc_test4/tst_parallel_zlib.c +++ b/nc_test4/tst_parallel_zlib.c @@ -103,8 +103,8 @@ main(int argc, char **argv) /* Create a parallel netcdf-4 file. */ /*nc_set_log_level(3);*/ - /* sprintf(file_name, "%s/%s", TEMP_LARGE, FILE); */ - sprintf(file_name, "%s", FILE); + /* snprintf(file_name, sizeof(file_name), "%s/%s", TEMP_LARGE, FILE); */ + snprintf(file_name, sizeof(file_name), "%s", FILE); if ((res = nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid))) ERR; /* Create three dimensions. */ diff --git a/nc_test4/tst_quantize.c b/nc_test4/tst_quantize.c index eb43c30437..c2694e7a9f 100644 --- a/nc_test4/tst_quantize.c +++ b/nc_test4/tst_quantize.c @@ -54,7 +54,7 @@ pf(float myf) uint32_t u; } fu; fu.f = myf; - sprintf(pf_str, "0x%x", fu.u); + snprintf(pf_str, sizeof(pf_str), "0x%x", fu.u); return pf_str; } @@ -67,7 +67,7 @@ pd(double myd) uint64_t u; } du; du.d = myd; - sprintf(pf_str, "0x%llx", (unsigned long long)du.u); + snprintf(pf_str, sizeof(pf_str), "0x%llx", (unsigned long long)du.u); return pf_str; } @@ -239,7 +239,7 @@ main(int argc, char **argv) printf("\t\t**** testing quantize algorithm %d...\n", quantize_mode[q]); for (t = 0; t < NTYPES; t++) { - sprintf(file_name, "%s_quantize_%d_type_%d.nc", TEST, quantize_mode[q], xtype[t]); + snprintf(file_name, sizeof(file_name), "%s_quantize_%d_type_%d.nc", TEST, quantize_mode[q], xtype[t]); #ifdef TESTNCZARR { char url[NC_MAX_FILENAME + 1]; diff --git a/nc_test4/tst_quantize_par.c b/nc_test4/tst_quantize_par.c index 9aad7c0a9b..efc0d87c85 100644 --- a/nc_test4/tst_quantize_par.c +++ b/nc_test4/tst_quantize_par.c @@ -50,7 +50,7 @@ pf(float myf) uint32_t u; } fu; fu.f = myf; - sprintf(pf_str, "0x%x", fu.u); + snprintf(pf_str, sizeof(pf_str), "0x%x", fu.u); return pf_str; } @@ -63,7 +63,7 @@ pd(double myd) uint64_t u; } du; du.d = myd; - sprintf(pf_str, "0x%lx", du.u); + snprintf(pf_str, sizeof(pf_str), "0x%lx", du.u); return pf_str; } diff --git a/nc_test4/tst_rename2.c b/nc_test4/tst_rename2.c index a86f7ffb1a..c366363fa4 100644 --- a/nc_test4/tst_rename2.c +++ b/nc_test4/tst_rename2.c @@ -62,7 +62,7 @@ main(int argc, char **argv) for (enddef_setting = 0; enddef_setting < NUM_ENDDEF_SETTINGS; enddef_setting++) { - sprintf(filename, "%s_%d_%d.nc", TEST_NAME, formats[format], + snprintf(filename, sizeof(filename), "%s_%d_%d.nc", TEST_NAME, formats[format], enddef_setting); /* Create file with three dims. */ @@ -110,7 +110,7 @@ main(int argc, char **argv) if (nc_set_default_format(formats[format], NULL)) ERR; - sprintf(filename, "%s_data_%d.nc", TEST_NAME, formats[format]); + snprintf(filename, sizeof(filename), "%s_data_%d.nc", TEST_NAME, formats[format]); /* Create file with three dims. */ if (nc_create(filename, 0, &ncid)) ERR; @@ -168,7 +168,7 @@ main(int argc, char **argv) char name[NC_MAX_NAME + 1]; /* Create file with dim and associated coordinate var. */ - sprintf(file_name, "%s_sync.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s_sync.nc", TEST_NAME); if (nc_create(file_name, NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; if (nc_def_dim(ncid, DIM_NAME_END, DIM1_LEN, &dimid)) ERR; if (nc_def_var(ncid, DIM_NAME_END, NC_INT, NDIM1, &dimid, &varid)) ERR; @@ -201,7 +201,7 @@ main(int argc, char **argv) char name[NC_MAX_NAME + 1]; /* Create file with dim and associated coordinate var. */ - sprintf(file_name, "%s_sync.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s_sync.nc", TEST_NAME); if (nc_create(file_name, NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; if (nc_def_dim(ncid, DIM_NAME_START, DIM1_LEN, &dimid)) ERR; if (nc_def_var(ncid, VAR_NAME_START, NC_INT, NDIM1, &dimid, &varid)) ERR; @@ -244,7 +244,7 @@ main(int argc, char **argv) char file_name[NC_MAX_NAME + 1]; /* Create file with dim and associated coordinate var. */ - sprintf(file_name, "%s_non_coord_to_dim.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s_non_coord_to_dim.nc", TEST_NAME); if (nc_create(file_name, NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; if (nc_def_dim(ncid, D1_NAME, DIM1_LEN, &dimid1)) ERR; if (nc_def_dim(ncid, D2_NAME, DIM1_LEN, &dimid2)) ERR; @@ -280,7 +280,7 @@ main(int argc, char **argv) char file_name[NC_MAX_NAME + 1]; /* Create file with two scalar vars. */ - sprintf(file_name, "%s_rename_affect_varid_order.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s_rename_affect_varid_order.nc", TEST_NAME); if (nc_create(file_name, NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; if (nc_def_var(ncid, D1_NAME, NC_INT, 0, NULL, &varid1)) ERR; if (nc_def_var(ncid, D2_NAME, NC_INT, 0, NULL, &varid2)) ERR; diff --git a/nc_test4/tst_rename3.c b/nc_test4/tst_rename3.c index 6f8c95ae67..e17af66344 100644 --- a/nc_test4/tst_rename3.c +++ b/nc_test4/tst_rename3.c @@ -39,7 +39,7 @@ main(int argc, char **argv) char file_name[NC_MAX_NAME + 1]; /* Create file with two scalar vars. */ - sprintf(file_name, "%s_coord_to_non_coord.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s_coord_to_non_coord.nc", TEST_NAME); if (nc_create(file_name, NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; if (nc_def_var(ncid, D1_NAME, NC_INT, 0, NULL, &varid1)) ERR; if (nc_def_var(ncid, D2_NAME, NC_INT, 0, NULL, &varid2)) ERR; @@ -67,7 +67,7 @@ main(int argc, char **argv) char file_name[NC_MAX_NAME + 1]; /* Create file with two dims and associated coordinate vars. */ - sprintf(file_name, "%s_coord_to_non_coord.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s_coord_to_non_coord.nc", TEST_NAME); if (nc_create(file_name, NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; if (nc_def_dim(ncid, D1_NAME, DIM1_LEN, &dimid1)) ERR; if (nc_def_dim(ncid, D2_NAME, DIM1_LEN, &dimid2)) ERR; @@ -121,7 +121,7 @@ main(int argc, char **argv) char file_name[NC_MAX_NAME + 1]; /* Create file with dim and associated coordinate var. */ - sprintf(file_name, "%s_non_coord_to_dim.nc", TEST_NAME); + snprintf(file_name, sizeof(file_name), "%s_non_coord_to_dim.nc", TEST_NAME); if (nc_create(file_name, NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &ncid)) ERR; if (nc_def_dim(ncid, D1_NAME, DIM1_LEN, &dimid1)) ERR; if (nc_def_dim(ncid, D2_NAME, DIM1_LEN, &dimid2)) ERR; diff --git a/nc_test4/tst_simplerw_coll_r.c b/nc_test4/tst_simplerw_coll_r.c index bf6637df69..b91d2a383d 100644 --- a/nc_test4/tst_simplerw_coll_r.c +++ b/nc_test4/tst_simplerw_coll_r.c @@ -236,7 +236,7 @@ main(int argc, char **argv) } /* Create a file name. */ - sprintf(file_name, "%s_type_%d_fv_%d.nc", TEST_NAME, test_type[tt], fv); + snprintf(file_name, sizeof(file_name), "%s_type_%d_fv_%d.nc", TEST_NAME, test_type[tt], fv); /* Create a parallel netcdf-4 file. */ if (nc_create_par(file_name, NC_NETCDF4, comm, info, &ncid)) ERR; diff --git a/nc_test4/tst_strings.c b/nc_test4/tst_strings.c index 301ca6cd79..ed78351a55 100644 --- a/nc_test4/tst_strings.c +++ b/nc_test4/tst_strings.c @@ -440,7 +440,7 @@ main(int argc, char **argv) char **string_fillp = dim_combo == 3 ? my_string_fill : &default_fill; char *data_in; - sprintf(filename, "%s_dim_combo_%d.nc", TEST_NAME, dim_combo); + snprintf(filename, sizeof(filename), "%s_dim_combo_%d.nc", TEST_NAME, dim_combo); if (nc_create(filename, NC_NETCDF4, &ncid)) ERR; /* Create an array of strings for the Universal Declaraion of Human Rights. */ diff --git a/nc_test4/tst_types.c b/nc_test4/tst_types.c index d4d82c30f6..f565dda33e 100644 --- a/nc_test4/tst_types.c +++ b/nc_test4/tst_types.c @@ -49,7 +49,7 @@ create_test_file(char *filename, int *varid, int *ncid) for (type = 0; type < NUM_TYPES; type++) { /* Create a var... */ - sprintf(varname, "var_%d", type); + snprintf(varname, sizeof(varname), "var_%d", type); if (nc_def_var(*ncid, varname, type + NC_UBYTE, 1, &dimid, &varid[type])) ERR; } return 0; @@ -101,7 +101,7 @@ int main(int argc, char *argv[]) for (type = 0; type < NUM_TYPES; type++) { /* Create a var... */ - sprintf(varname, "var_%d", type); + snprintf(varname, sizeof(varname), "var_%d", type); if (nc_def_var(ncid1, varname, type + NC_UBYTE, 1, &dimid, &varid[type])) ERR; if (nc_inq_type_equal(ncid1, type + NC_UBYTE, ncid2, type + NC_UBYTE, &equal)) ERR; if (!equal) ERR; diff --git a/nc_test4/tst_vars.c b/nc_test4/tst_vars.c index f94fc0dcbb..0124291e94 100644 --- a/nc_test4/tst_vars.c +++ b/nc_test4/tst_vars.c @@ -1279,7 +1279,7 @@ main(int argc, char **argv) /* Create a large number of variables. */ for (i = 0; i < NUM_VARS; i++) { - sprintf(varname, "a_%d", i); + snprintf(varname, sizeof(varname), "a_%d", i); if (nc_def_var(ncid, varname, NC_FLOAT, 1, dimids, &varids[i])) { ERR; break; diff --git a/nc_test4/tst_vars3.c b/nc_test4/tst_vars3.c index 19d51de13a..f0420bccda 100644 --- a/nc_test4/tst_vars3.c +++ b/nc_test4/tst_vars3.c @@ -314,7 +314,7 @@ main(int argc, char **argv) if (nc_def_dim(ncid, D0_NAME3, NC_UNLIMITED, &dimid)) ERR; for (v = 0; v < NUM_VARS; v++) { - sprintf(var_name, "var_%d", v); + snprintf(var_name, sizeof(var_name), "var_%d", v); if (nc_def_var(ncid, var_name, NC_INT, 1, &dimid, &varid)) ERR_RET; if (nc_set_var_chunk_cache(ncid, varid, 0, 0, 0.75)) ERR_RET; } diff --git a/ncdump/ncvalidator.c b/ncdump/ncvalidator.c index f3661269cb..971629085d 100644 --- a/ncdump/ncvalidator.c +++ b/ncdump/ncvalidator.c @@ -815,7 +815,7 @@ compute_var_shape(NC *ncp) ncp->recsize = 0; for (i=0; ivars.ndefined; i++) { - sprintf(xloc,"var %s:",ncp->vars.value[i]->name); + snprintf(xloc,sizeof(xloc),"var %s:",ncp->vars.value[i]->name); /* check if dimids are valid */ for (j=0; jvars.value[i]->ndims; j++) { if (ncp->vars.value[i]->dimids[j] < 0) { @@ -1464,7 +1464,7 @@ val_get_NC_attr(int fd, return status; } - sprintf(xloc,"%s \"%s\"",loc,name); + snprintf(xloc,sizeof(xloc),"%s \"%s\"",loc,name); err = val_get_nc_type(fd, gbp, &xtype, xloc); if (err != NC_NOERR) { if (name != NULL) free(name); @@ -1575,7 +1575,7 @@ val_get_NC_attrarray(int fd, } #endif } else { - sprintf(xloc, "%s attribute", loc); + snprintf(xloc, sizeof(xloc), "%s attribute", loc); if (tag != NC_ATTRIBUTE) { if (verbose) printf("Error @ [0x%8.8zx]:\n", tag_err_addr); if (verbose) printf("\t%s: Invalid NC component tag (%d), expecting NC_ATTRIBUTE (%d)\n",xloc,tag,NC_ATTRIBUTE); @@ -1698,7 +1698,7 @@ val_get_NC_var(int fd, if (trace) printf("\t\tname = \"%s\"\n", name); /* read number of dimensions */ - sprintf(xloc,"%s \"%s\"",loc,name); + snprintf(xloc,sizeof(xloc),"%s \"%s\"",loc,name); err_addr = ERR_ADDR; err = hdr_get_NON_NEG(fd, gbp, &ndims); if (err != NC_NOERR) { @@ -1760,7 +1760,7 @@ val_get_NC_var(int fd, /* var = name nelems [dimid ...] vatt_list nc_type vsize begin * ^^^^^^^^^ */ - sprintf(xloc,"%s \"%s\"",loc,name); + snprintf(xloc,sizeof(xloc),"%s \"%s\"",loc,name); err = val_get_NC_attrarray(fd, gbp, &varp->attrs, xloc); if (err != NC_NOERR && err != NC_ENULLPAD) { free_NC_var(varp); diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index 1e25f2daf6..7c0abc47d9 100755 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -1033,7 +1033,7 @@ dumpdatanode(OClink link, OCdatanode datanode, size_t count, void* memory, NCbyt ncbytescat(path,id); if(entry->rank > 0) { for(i=0;irank;i++) { - sprintf(tmp,"[%lu]",(unsigned long)entry->indices[i]); + snprintf(tmp,sizeof(tmp),"[%lu]",(unsigned long)entry->indices[i]); ncbytescat(path,tmp); } } @@ -1047,7 +1047,7 @@ dumpdatanode(OClink link, OCdatanode datanode, size_t count, void* memory, NCbyt case OC_Sequence: ncbytescat(path,"/"); ncbytescat(path,id); - sprintf(tmp,"[%lu]",(unsigned long)entry->indices[0]); + snprintf(tmp,sizeof(tmp),"[%lu]",(unsigned long)entry->indices[0]); ncbytescat(path,tmp); break; diff --git a/ncgen/cmldata.c b/ncgen/cmldata.c index f2034b495d..07931d0dc9 100644 --- a/ncgen/cmldata.c +++ b/ncgen/cmldata.c @@ -288,37 +288,37 @@ xconst(Constant* ci) return result; } break; case NC_BYTE: - sprintf(tmp,"%hhd",ci->value.int8v); + snprintf(tmp,sizeof(tmp),"%hhd",ci->value.int8v); break; case NC_SHORT: - sprintf(tmp,"%hd",ci->value.int16v); + snprintf(tmp,sizeof(tmp),"%hd",ci->value.int16v); break; case NC_INT: - sprintf(tmp,"%d",ci->value.int32v); + snprintf(tmp,sizeof(tmp),"%d",ci->value.int32v); break; case NC_FLOAT: - sprintf(tmp,"%.8g",ci->value.floatv); + snprintf(tmp,sizeof(tmp),"%.8g",ci->value.floatv); break; case NC_DOUBLE: - sprintf(tmp,"%.16g",ci->value.doublev); + snprintf(tmp,sizeof(tmp),"%.16g",ci->value.doublev); break; case NC_UBYTE: - sprintf(tmp,"%hhu",ci->value.uint8v); + snprintf(tmp,sizeof(tmp),"%hhu",ci->value.uint8v); break; case NC_USHORT: - sprintf(tmp,"%hu",ci->value.uint16v); + snprintf(tmp,sizeof(tmp),"%hu",ci->value.uint16v); break; case NC_UINT: - sprintf(tmp,"%uU",ci->value.uint32v); + snprintf(tmp,sizeof(tmp),"%uU",ci->value.uint32v); break; case NC_INT64: - sprintf(tmp,"%lldLL",ci->value.int64v); + snprintf(tmp,sizeof(tmp),"%lldLL",ci->value.int64v); break; case NC_UINT64: - sprintf(tmp,"%lluLLU",ci->value.uint64v); + snprintf(tmp,sizeof(tmp),"%lluLLU",ci->value.uint64v); break; case NC_ECONST: - sprintf(tmp,"%s",cname(ci->value.enumv)); + snprintf(tmp,sizeof(tmp),"%s",cname(ci->value.enumv)); break; case NC_STRING: { diff --git a/ncgen/cvt.c b/ncgen/cvt.c index 8224ea6ad7..29995d82f1 100644 --- a/ncgen/cvt.c +++ b/ncgen/cvt.c @@ -459,57 +459,57 @@ case CASE(NC_STRING,NC_STRING): /* What is the proper conversion for T->STRING?*/ case CASE(NC_CHAR,NC_STRING): - sprintf(stmp,"%c",src->value.charv); + snprintf(stmp, sizeof(stmp),"%c",src->value.charv); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_BYTE,NC_STRING): - sprintf(stmp,"%hhd",src->value.uint8v); + snprintf(stmp, sizeof(stmp),"%hhd",src->value.uint8v); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_UBYTE,NC_STRING): - sprintf(stmp,"%hhu",src->value.uint8v); + snprintf(stmp, sizeof(stmp),"%hhu",src->value.uint8v); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_USHORT,NC_STRING): - sprintf(stmp,"%hu",src->value.uint16v); + snprintf(stmp, sizeof(stmp),"%hu",src->value.uint16v); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_UINT,NC_STRING): - sprintf(stmp,"%u",src->value.uint32v); + snprintf(stmp, sizeof(stmp),"%u",src->value.uint32v); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_UINT64,NC_STRING): - sprintf(stmp,"%llu",src->value.uint64v); + snprintf(stmp, sizeof(stmp),"%llu",src->value.uint64v); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_SHORT,NC_STRING): - sprintf(stmp,"%hd",src->value.int16v); + snprintf(stmp, sizeof(stmp),"%hd",src->value.int16v); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_INT,NC_STRING): - sprintf(stmp,"%d",src->value.int32v); + snprintf(stmp, sizeof(stmp),"%d",src->value.int32v); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_INT64,NC_STRING): - sprintf(stmp,"%lld",src->value.int64v); + snprintf(stmp, sizeof(stmp),"%lld",src->value.int64v); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_FLOAT,NC_STRING): - sprintf(stmp,"%.8g",src->value.floatv); + snprintf(stmp, sizeof(stmp),"%.8g",src->value.floatv); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; case CASE(NC_DOUBLE,NC_STRING): - sprintf(stmp,"%.8g",src->value.doublev); + snprintf(stmp, sizeof(stmp),"%.8g",src->value.doublev); tmp.stringv.len = nulllen(stmp); tmp.stringv.stringv = nulldup(stmp); break; diff --git a/ncgen/dump.c b/ncgen/dump.c index bd4d775b86..024f92c41d 100644 --- a/ncgen/dump.c +++ b/ncgen/dump.c @@ -83,7 +83,7 @@ bufdump(Datalist* list, Bytebuffer* buf) dumpdataprim(dp,buf); } else { char tmp[64]; - sprintf(tmp,"?%d? ",dp->nctype); + snprintf(tmp,sizeof(tmp),"?%d? ",dp->nctype); bbCat(buf,tmp); } break; } @@ -104,47 +104,47 @@ dumpdataprim(NCConstant* ci, Bytebuffer* buf) bbCat(buf,"'"); } break; case NC_BYTE: - sprintf(tmp,"%hhd",ci->value.int8v); + snprintf(tmp,sizeof(tmp),"%hhd",ci->value.int8v); bbCat(buf,tmp); break; case NC_SHORT: - sprintf(tmp,"%hd",ci->value.int16v); + snprintf(tmp,sizeof(tmp),"%hd",ci->value.int16v); bbCat(buf,tmp); break; case NC_INT: - sprintf(tmp,"%d",ci->value.int32v); + snprintf(tmp,sizeof(tmp),"%d",ci->value.int32v); bbCat(buf,tmp); break; case NC_FLOAT: - sprintf(tmp,"%g",ci->value.floatv); + snprintf(tmp,sizeof(tmp),"%g",ci->value.floatv); bbCat(buf,tmp); break; case NC_DOUBLE: - sprintf(tmp,"%lg",ci->value.doublev); + snprintf(tmp,sizeof(tmp),"%lg",ci->value.doublev); bbCat(buf,tmp); break; case NC_UBYTE: - sprintf(tmp,"%hhu",ci->value.int8v); + snprintf(tmp,sizeof(tmp),"%hhu",ci->value.int8v); bbCat(buf,tmp); break; case NC_USHORT: - sprintf(tmp,"%hu",ci->value.uint16v); + snprintf(tmp,sizeof(tmp),"%hu",ci->value.uint16v); bbCat(buf,tmp); break; case NC_UINT: - sprintf(tmp,"%u",ci->value.uint32v); + snprintf(tmp,sizeof(tmp),"%u",ci->value.uint32v); bbCat(buf,tmp); break; case NC_INT64: - sprintf(tmp,"%lld",ci->value.int64v); + snprintf(tmp,sizeof(tmp),"%lld",ci->value.int64v); bbCat(buf,tmp); break; case NC_UINT64: - sprintf(tmp,"%llu",ci->value.uint64v); + snprintf(tmp,sizeof(tmp),"%llu",ci->value.uint64v); bbCat(buf,tmp); break; case NC_ECONST: - sprintf(tmp,"%s",ci->value.enumv->fqn); + snprintf(tmp,sizeof(tmp),"%s",ci->value.enumv->fqn); bbCat(buf,tmp); break; case NC_STRING: diff --git a/ncgen/f77data.c b/ncgen/f77data.c index e4863af2ac..28521890a3 100644 --- a/ncgen/f77data.c +++ b/ncgen/f77data.c @@ -35,26 +35,26 @@ f77_constant(Generator* generator, Symbol* sym, NCConstant* ci, Bytebuffer* code case NC_CHAR: if(ci->value.charv == '\'') - sprintf(tmp,"'\\''"); + snprintf(tmp,sizeof(tmp),"'\\''"); else - sprintf(tmp,"'%c'",ci->value.charv); + snprintf(tmp,sizeof(tmp),"'%c'",ci->value.charv); break; case NC_BYTE: - sprintf(tmp,"%hhd",ci->value.int8v); + snprintf(tmp,sizeof(tmp),"%hhd",ci->value.int8v); break; case NC_SHORT: - sprintf(tmp,"%hd",ci->value.int16v); + snprintf(tmp,sizeof(tmp),"%hd",ci->value.int16v); break; case NC_INT: - sprintf(tmp,"%d",ci->value.int32v); + snprintf(tmp,sizeof(tmp),"%d",ci->value.int32v); break; case NC_FLOAT: - sprintf(tmp,"%.8g",ci->value.floatv); + snprintf(tmp,sizeof(tmp),"%.8g",ci->value.floatv); break; case NC_DOUBLE: { char* p = tmp; /* FORTRAN requires e|E->D */ - sprintf(tmp,"%.16g",ci->value.doublev); + snprintf(tmp,sizeof(tmp),"%.16g",ci->value.doublev); while(*p) {if(*p == 'e' || *p == 'E') {*p = 'D';}; p++;} } break; case NC_STRING: diff --git a/ncgen/gencml.c b/ncgen/gencml.c index b88b080175..912cc49751 100644 --- a/ncgen/gencml.c +++ b/ncgen/gencml.c @@ -316,7 +316,7 @@ definextype(Symbol* tsym) if(j > 0) bbCat(dimbuf," "); dim = efield->typ.dimset.dimsyms[j]; ASSERT(dim->dim.isconstant); - sprintf(tmp,"%lu",(unsigned long)dim->dim.size); + snprintf(tmp,sizeof(tmp),"%lu",(unsigned long)dim->dim.size); bbCat(dimbuf,tmp); } bbCat(dimbuf,"\""); diff --git a/ncgen/genjjni.c b/ncgen/genjjni.c index 86c8682b4b..03e0166f7c 100644 --- a/ncgen/genjjni.c +++ b/ncgen/genjjni.c @@ -838,7 +838,7 @@ genjjni_deftype(Symbol* tsym) char tmp[256]; Symbol* e = efield->typ.dimset.dimsyms[j]; ASSERT(e->dim.isconstant); - sprintf(tmp,"%u",e->dim.size); + snprintf(tmp,sizeof(tmp),"%u",e->dim.size); strcat(stmt,(j==0?"":", ")); strcat(stmt,tmp); } @@ -849,7 +849,7 @@ genjjni_deftype(Symbol* tsym) Symbol* efield = (Symbol*)listget(tsym->subnodes,i); char tmp[1024]; ASSERT(efield->subclass == NC_FIELD); - sprintf(tmp,"%lu",efield->typ.offset); + snprintf(tmp,sizeof(tmp),"%lu",efield->typ.offset); if(efield->typ.dimset.ndims > 0){ nprintf(stmt,sizeof(stmt),"check_err(nc_insert_array_compound(%s, %s, \"%s\", %s, %s, %d, %s_dims));", jgroupncid(tsym->container), @@ -1449,39 +1449,39 @@ jconst(Constant* ci) } break; case NC_BYTE: - sprintf(tmp,"%hhd",ci->value.int8v); + snprintf(tmp,sizeof(tmp),"%hhd",ci->value.int8v); break; case NC_SHORT: - sprintf(tmp,"%hd",ci->value.int16v); + snprintf(tmp,sizeof(tmp),"%hd",ci->value.int16v); break; case NC_INT: - sprintf(tmp,"%d",ci->value.int32v); + snprintf(tmp,sizeof(tmp),"%d",ci->value.int32v); break; case NC_FLOAT: - sprintf(tmp,"%.8g",ci->value.floatv); + snprintf(tmp,sizeof(tmp),"%.8g",ci->value.floatv); break; case NC_DOUBLE: - sprintf(tmp,"%.16g",ci->value.doublev); + snprintf(tmp,sizeof(tmp),"%.16g",ci->value.doublev); break; case NC_UBYTE: - sprintf(tmp,"%hhu",ci->value.uint8v); + snprintf(tmp,sizeof(tmp),"%hhu",ci->value.uint8v); break; case NC_USHORT: - sprintf(tmp,"%hu",ci->value.uint16v); + snprintf(tmp,sizeof(tmp),"%hu",ci->value.uint16v); break; case NC_UINT: - sprintf(tmp,"%uL",ci->value.uint32v); /* upgrade to long */ + snprintf(tmp,sizeof(tmp),"%uL",ci->value.uint32v); /* upgrade to long */ break; case NC_INT64: - sprintf(tmp,"%lldL",ci->value.int64v); + snprintf(tmp,sizeof(tmp),"%lldL",ci->value.int64v); break; case NC_UINT64: { /* HACK to handle unsigned values */ long long l = (long)ci->value.uint64v; - sprintf(tmp,"%lldL",l); + snprintf(tmp,sizeof(tmp),"%lldL",l); } break; case NC_ECONST: - sprintf(tmp,"%s",jname(ci->value.enumv)); + snprintf(tmp,sizeof(tmp),"%s",jname(ci->value.enumv)); break; case NC_STRING: { diff --git a/ncgen/jdatajni.c b/ncgen/jdatajni.c index f5edb8d7e0..9faf6ae4f1 100644 --- a/ncgen/jdatajni.c +++ b/ncgen/jdatajni.c @@ -244,13 +244,13 @@ genjjni_data(Symbol* tsym, Datasrc* datasrc, Datalist* fillsrc, semerror(srcline(datasrc),"Vlen data must be enclosed in {..}"); } cp = srcnext(datasrc); - sprintf(vlenname,"vlen_%u",cp->value.compoundv->vlen.uid); + snprintf(vlenname,sizeof(vlenname),"vlen_%u",cp->value.compoundv->vlen.uid); /* Use special alignment */ nprintf(stmt,sizeof(stmt),"Memory.align(%s,%d);\n", vlenname,jvlenalignment()); bbCat(databuf,stmt); /* generate and store the nc_vlen_t instance*/ - sprintf(stmt,"Memory.put_vlen(%s,new long[]{%lld, %s_addr});\n", + snprintf(stmt,sizeof(stmt),"Memory.put_vlen(%s,new long[]{%lld, %s_addr});\n", memname, (long long)cp->value.compoundv->vlen.count, vlenname); @@ -336,7 +336,7 @@ genjjni_vlendata(List* vlenconstants, Bytebuffer* databuf) Symbol* basetype = tsym->typ.basetype; int typecode = basetype->typ.typecode; - sprintf(memname,"vlen_%u",cmpd->value.compoundv->vlen.uid); + snprintf(memname,sizeof(memname),"vlen_%u",cmpd->value.compoundv->vlen.uid); count = 0; vlensrc = datalist2src(cmpd->value.compoundv); @@ -362,7 +362,7 @@ genjjni_vlendata(List* vlenconstants, Bytebuffer* databuf) bbCat(databuf,bbContents(vlenbuf)); } /* Now store the address */ - sprintf(tmp,"%s_addr = Memory.address(%s);\n", + snprintf(tmp,sizeof(tmp),"%s_addr = Memory.address(%s);\n", memname,memname); bbCat(databuf,tmp); } @@ -380,11 +380,11 @@ genjjni_vlenconstants(List* vlenconstants, Bytebuffer* databuf) for(i=0;ivalue.compoundv->vlen.uid); + snprintf(memname,sizeof(memname),"vlen_%u",cmpd->value.compoundv->vlen.uid); /* Define the memory buffer and a place to stick its address */ - sprintf(tmp,"static long %s = Memory.create();\n",memname); + snprintf(tmp,sizeof(tmp),"static long %s = Memory.create();\n",memname); bbCat(databuf,tmp); - sprintf(tmp,"static long %s_addr = 0L;\n",memname); + snprintf(tmp,sizeof(tmp),"static long %s_addr = 0L;\n",memname); bbCat(databuf,tmp); } } diff --git a/ncgen/ncgen.l b/ncgen/ncgen.l index c46fb38634..963ecefcca 100644 --- a/ncgen/ncgen.l +++ b/ncgen/ncgen.l @@ -243,7 +243,7 @@ yytext[MAXTRST-1] = '\0'; */ len = unescape((char *)yytext+1,yyleng-2,!ISIDENT,&s); if(len < 0) { - sprintf(errstr,"Illegal character: %s",yytext); + snprintf(errstr, sizeof(errstr),"Illegal character: %s",yytext); yyerror(errstr); } bbClear(lextext); @@ -411,7 +411,7 @@ NIL|nil|Nil { /* capture the tag string */ tag = collecttag(pos,&stag); if(tag == NC_NAT) { - sprintf(errstr,"Illegal integer suffix: %s",stag); + snprintf(errstr, sizeof(errstr),"Illegal integer suffix: %s",stag); yyerror(errstr); goto done; } @@ -433,13 +433,13 @@ NIL|nil|Nil { radix = 10; if(isneg && hasU) { - sprintf(errstr,"Unsigned integer cannot be signed: %s",ncgtext); + snprintf(errstr, sizeof(errstr),"Unsigned integer cannot be signed: %s",ncgtext); yyerror(errstr); goto done; } uint64_val = parseULL(radix, pos,&fail); if(fail) { - sprintf(errstr,"integer constant out of range: %s",ncgtext); + snprintf(errstr, sizeof(errstr),"integer constant out of range: %s",ncgtext); yyerror(errstr); goto done; } @@ -453,7 +453,7 @@ NIL|nil|Nil { case NC_FORMAT_64BIT_OFFSET: case NC_FORMAT_NETCDF4_CLASSIC: if(nct > NC_INT) { - sprintf(errstr,"Illegal integer constant for classic format: %s",ncgtext); + snprintf(errstr, sizeof(errstr),"Illegal integer constant for classic format: %s",ncgtext); yyerror(errstr); goto done; } @@ -479,7 +479,7 @@ done: return 0; /* capture the tag string */ tag = collecttag(yytext,&stag); if(tag == NC_NAT) { - sprintf(errstr,"Illegal integer suffix: %s",stag); + snprintf(errstr, sizeof(errstr),"Illegal integer suffix: %s",stag); yyerror(errstr); //goto done; return 0; @@ -507,7 +507,7 @@ done: return 0; break; default: /* should never happen */ if (sscanf((char*)yytext, "%i", &uint32_val) != 1) { - sprintf(errstr,"bad unsigned int constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad unsigned int constant: %s",(char*)yytext); yyerror(errstr); } token = UINT_CONST; @@ -516,14 +516,14 @@ done: return 0; } {DBLNUMBER} { if (sscanf((char*)yytext, "%le", &double_val) != 1) { - sprintf(errstr,"bad long or double constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad long or double constant: %s",(char*)yytext); yyerror(errstr); } return lexdebug(DOUBLE_CONST); } {FLTNUMBER} { if (sscanf((char*)yytext, "%e", &float_val) != 1) { - sprintf(errstr,"bad float constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad float constant: %s",(char*)yytext); yyerror(errstr); } return lexdebug(FLOAT_CONST); @@ -535,7 +535,7 @@ done: return 0; \'\\[0-7][0-7][0-7]\' { int oct = unescapeoct(&yytext[2]); if(oct < 0) { - sprintf(errstr,"bad octal character constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad octal character constant: %s",(char*)yytext); yyerror(errstr); } byte_val = (unsigned int)oct; @@ -544,7 +544,7 @@ done: return 0; \'\\[xX][0-9a-fA-F][0-9a-fA-F]\' { int hex = unescapehex(&yytext[3]); if(byte_val < 0) { - sprintf(errstr,"bad hex character constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad hex character constant: %s",(char*)yytext); yyerror(errstr); } byte_val = (unsigned int)hex; @@ -647,7 +647,7 @@ makepath(char* text0) refsym = lookupingroup(NC_GRP,ident,container); if(!lastident) { if(refsym == NULL) { - sprintf(errstr,"Undefined or forward referenced group: %s",ident); + snprintf(errstr, sizeof(errstr),"Undefined or forward referenced group: %s",ident); yyerror(errstr); refsym = rootgroup; } else diff --git a/ncgen/ncgen.y b/ncgen/ncgen.y index 10f3e36ea9..c8c2d5ac62 100644 --- a/ncgen/ncgen.y +++ b/ncgen/ncgen.y @@ -650,7 +650,7 @@ fielddim: { /* Anonymous integer dimension. Can only occur in type definitions*/ char anon[32]; - sprintf(anon,"const%u",uint32_val); + snprintf(anon, sizeof(anon),"const%u",uint32_val); $$ = install(anon); $$->objectclass = NC_DIM; $$->dim.isconstant = 1; @@ -664,7 +664,7 @@ fielddim: derror("field dimension must be positive"); YYABORT; } - sprintf(anon,"const%d",int32_val); + snprintf(anon, sizeof(anon),"const%d",int32_val); $$ = install(anon); $$->objectclass = NC_DIM; $$->dim.isconstant = 1; diff --git a/ncgen/ncgenl.c b/ncgen/ncgenl.c index 6ea3a2270e..af03296e16 100644 --- a/ncgen/ncgenl.c +++ b/ncgen/ncgenl.c @@ -2081,7 +2081,7 @@ yytext[MAXTRST-1] = '\0'; */ len = unescape((char *)yytext+1,yyleng-2,!ISIDENT,&s); if(len < 0) { - sprintf(errstr,"Illegal character: %s",yytext); + snprintf(errstr,sizeof(errstr),"Illegal character: %s",yytext); yyerror(errstr); } bbClear(lextext); @@ -2385,7 +2385,7 @@ YY_RULE_SETUP /* capture the tag string */ tag = collecttag(pos,&stag); if(tag == NC_NAT) { - sprintf(errstr,"Illegal integer suffix: %s",stag); + snprintf(errstr,sizeof(errstr),"Illegal integer suffix: %s",stag); yyerror(errstr); goto done; } @@ -2407,13 +2407,13 @@ YY_RULE_SETUP radix = 10; if(isneg && hasU) { - sprintf(errstr,"Unsigned integer cannot be signed: %s",ncgtext); + snprintf(errstr,sizeof(errstr),"Unsigned integer cannot be signed: %s",ncgtext); yyerror(errstr); goto done; } uint64_val = parseULL(radix, pos,&fail); if(fail) { - sprintf(errstr,"integer constant out of range: %s",ncgtext); + snprintf(errstr,sizeof(errstr),"integer constant out of range: %s",ncgtext); yyerror(errstr); goto done; } @@ -2427,7 +2427,7 @@ YY_RULE_SETUP case NC_FORMAT_64BIT_OFFSET: case NC_FORMAT_NETCDF4_CLASSIC: if(nct > NC_INT) { - sprintf(errstr,"Illegal integer constant for classic format: %s",ncgtext); + snprintf(errstr,sizeof(errstr),"Illegal integer constant for classic format: %s",ncgtext); yyerror(errstr); goto done; } @@ -2456,7 +2456,7 @@ YY_RULE_SETUP /* capture the tag string */ tag = collecttag(yytext,&stag); if(tag == NC_NAT) { - sprintf(errstr,"Illegal integer suffix: %s",stag); + snprintf(errstr,sizeof(errstr),"Illegal integer suffix: %s",stag); yyerror(errstr); //goto done; return 0; @@ -2484,7 +2484,7 @@ YY_RULE_SETUP break; default: /* should never happen */ if (sscanf((char*)yytext, "%i", &uint32_val) != 1) { - sprintf(errstr,"bad unsigned int constant: %s",(char*)yytext); + snprintf(errstr,sizeof(errstr),"bad unsigned int constant: %s",(char*)yytext); yyerror(errstr); } token = UINT_CONST; @@ -2497,7 +2497,7 @@ YY_RULE_SETUP #line 517 "ncgen.l" { if (sscanf((char*)yytext, "%le", &double_val) != 1) { - sprintf(errstr,"bad long or double constant: %s",(char*)yytext); + snprintf(errstr,sizeof(errstr),"bad long or double constant: %s",(char*)yytext); yyerror(errstr); } return lexdebug(DOUBLE_CONST); @@ -2508,7 +2508,7 @@ YY_RULE_SETUP #line 524 "ncgen.l" { if (sscanf((char*)yytext, "%e", &float_val) != 1) { - sprintf(errstr,"bad float constant: %s",(char*)yytext); + snprintf(errstr,sizeof(errstr),"bad float constant: %s",(char*)yytext); yyerror(errstr); } return lexdebug(FLOAT_CONST); @@ -2529,7 +2529,7 @@ YY_RULE_SETUP { int oct = unescapeoct(&yytext[2]); if(oct < 0) { - sprintf(errstr,"bad octal character constant: %s",(char*)yytext); + snprintf(errstr,sizeof(errstr),"bad octal character constant: %s",(char*)yytext); yyerror(errstr); } byte_val = (unsigned int)oct; @@ -2542,7 +2542,7 @@ YY_RULE_SETUP { int hex = unescapehex(&yytext[3]); if(byte_val < 0) { - sprintf(errstr,"bad hex character constant: %s",(char*)yytext); + snprintf(errstr,sizeof(errstr),"bad hex character constant: %s",(char*)yytext); yyerror(errstr); } byte_val = (unsigned int)hex; @@ -3681,7 +3681,7 @@ makepath(char* text0) refsym = lookupingroup(NC_GRP,ident,container); if(!lastident) { if(refsym == NULL) { - sprintf(errstr,"Undefined or forward referenced group: %s",ident); + snprintf(errstr,sizeof(errstr),"Undefined or forward referenced group: %s",ident); yyerror(errstr); refsym = rootgroup; } else diff --git a/ncgen/ncgeny.c b/ncgen/ncgeny.c index 105a221b4c..ae2c0a81e6 100644 --- a/ncgen/ncgeny.c +++ b/ncgen/ncgeny.c @@ -1543,7 +1543,7 @@ yysyntax_error (YYPTRDIFF_T *yymsg_alloc, char **yymsg, return -1; } - /* Avoid sprintf, as that infringes on the user's name space. + /* Avoid snprintf, as that infringes on the user's name space. Don't have undefined behavior even if the translation produced a string with the wrong number of "%s"s. */ { @@ -2409,7 +2409,7 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); { /* Anonymous integer dimension. Can only occur in type definitions*/ char anon[32]; - sprintf(anon,"const%u",uint32_val); + snprintf(anon,sizeof(anon),"const%u",uint32_val); (yyval.sym) = install(anon); (yyval.sym)->objectclass = NC_DIM; (yyval.sym)->dim.isconstant = 1; @@ -2427,7 +2427,7 @@ fprintf(stderr,"dimension: %s = UNLIMITED\n",(yyvsp[-2].sym)->name); derror("field dimension must be positive"); YYABORT; } - sprintf(anon,"const%d",int32_val); + snprintf(anon,sizeof(anon),"const%d",int32_val); (yyval.sym) = install(anon); (yyval.sym)->objectclass = NC_DIM; (yyval.sym)->dim.isconstant = 1; diff --git a/ncgen/odom.c b/ncgen/odom.c index 1f96828d65..93e9a34ec0 100644 --- a/ncgen/odom.c +++ b/ncgen/odom.c @@ -70,7 +70,7 @@ odometerprint(Odometer* odom) strcat(line,"[]"); } else for(i=0;irank;i++) { int ioffset = i + odom->offset; - sprintf(tmp,"[%lu/%lu..%lu:%lu]", + snprintf(tmp,sizeof(tmp),"[%lu/%lu..%lu:%lu]", (unsigned long)odom->origin->index[ioffset], (unsigned long)odom->origin->start[ioffset], (unsigned long)odom->origin->declsize[ioffset], diff --git a/ncgen3/genlib.c b/ncgen3/genlib.c index 1e728502ec..ead98b08df 100644 --- a/ncgen3/genlib.c +++ b/ncgen3/genlib.c @@ -225,7 +225,7 @@ gen_c( cline("}"); cline(""); cline("int"); - sprintf(stmnt, "main() {\t\t\t/* create %s */", filename); + snprintf(stmnt, sizeof(stmnt), "main() {\t\t\t/* create %s */", filename); cline(stmnt); /* create necessary declarations */ @@ -237,7 +237,7 @@ gen_c( cline(""); cline(" /* dimension ids */"); for (idim = 0; idim < ndims; idim++) { - sprintf(stmnt, " int %s_dim;", dims[idim].lname); + snprintf(stmnt, sizeof(stmnt), " int %s_dim;", dims[idim].lname); cline(stmnt); } @@ -245,10 +245,10 @@ gen_c( cline(" /* dimension lengths */"); for (idim = 0; idim < ndims; idim++) { if (dims[idim].size == NC_UNLIMITED) { - sprintf(stmnt, " size_t %s_len = NC_UNLIMITED;", + snprintf(stmnt, sizeof(stmnt), " size_t %s_len = NC_UNLIMITED;", dims[idim].lname); } else { - sprintf(stmnt, " size_t %s_len = %lu;", + snprintf(stmnt, sizeof(stmnt), " size_t %s_len = %lu;", dims[idim].lname, (unsigned long) dims[idim].size); } @@ -265,14 +265,14 @@ gen_c( cline(""); cline(" /* variable ids */"); for (ivar = 0; ivar < nvars; ivar++) { - sprintf(stmnt, " int %s_id;", vars[ivar].lname); + snprintf(stmnt, sizeof(stmnt), " int %s_id;", vars[ivar].lname); cline(stmnt); } cline(""); cline(" /* rank (number of dimensions) for each variable */"); for (ivar = 0; ivar < nvars; ivar++) { - sprintf(stmnt, "# define RANK_%s %d", vars[ivar].lname, + snprintf(stmnt, sizeof(stmnt), "# define RANK_%s %d", vars[ivar].lname, vars[ivar].ndims); cline(stmnt); } @@ -281,7 +281,7 @@ gen_c( cline(" /* variable shapes */"); for (ivar = 0; ivar < nvars; ivar++) { if (vars[ivar].ndims > 0) { - sprintf(stmnt, " int %s_dims[RANK_%s];", + snprintf(stmnt, sizeof(stmnt), " int %s_dims[RANK_%s];", vars[ivar].lname, vars[ivar].lname); cline(stmnt); } @@ -302,7 +302,7 @@ gen_c( cline(" /* attribute vectors */"); for (iatt = 0; iatt < natts; iatt++) { if (atts[iatt].type != NC_CHAR) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " %s %s_%s[%lu];", ncatype(atts[iatt].type), atts[iatt].var == -1 ? "cdf" : vars[atts[iatt].var].lname, @@ -318,20 +318,20 @@ gen_c( cline(" /* enter define mode */"); if (!cmode_modifier) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_create(\"%s\", NC_CLOBBER, &ncid);", filename); } else if (cmode_modifier & NC_64BIT_OFFSET) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_create(\"%s\", NC_CLOBBER|NC_64BIT_OFFSET, &ncid);", filename); #ifdef USE_NETCDF4 } else if (cmode_modifier & NC_CLASSIC_MODEL) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_create(\"%s\", NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &ncid);", filename); } else if (cmode_modifier & NC_NETCDF4) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_create(\"%s\", NC_CLOBBER|NC_NETCDF4, &ncid);", filename); #endif @@ -347,7 +347,7 @@ gen_c( cline(" /* define dimensions */"); } for (idim = 0; idim < ndims; idim++) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_def_dim(ncid, \"%s\", %s_len, &%s_dim);", dims[idim].name, dims[idim].lname, dims[idim].lname); cline(stmnt); @@ -361,7 +361,7 @@ gen_c( for (ivar = 0; ivar < nvars; ivar++) { cline(""); for (idim = 0; idim < vars[ivar].ndims; idim++) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " %s_dims[%d] = %s_dim;", vars[ivar].lname, idim, @@ -369,7 +369,7 @@ gen_c( cline(stmnt); } if (vars[ivar].ndims > 0) { /* a dimensioned variable */ - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_def_var(ncid, \"%s\", %s, RANK_%s, %s_dims, &%s_id);", vars[ivar].name, nctype(vars[ivar].type), @@ -377,7 +377,7 @@ gen_c( vars[ivar].lname, vars[ivar].lname); } else { /* a scalar */ - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_def_var(ncid, \"%s\", %s, RANK_%s, 0, &%s_id);", vars[ivar].name, nctype(vars[ivar].type), @@ -396,7 +396,7 @@ gen_c( for (iatt = 0; iatt < natts; iatt++) { if (atts[iatt].type == NC_CHAR) { /* string */ val_string = cstrstr((char *) atts[iatt].val, atts[iatt].len); - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_put_att_text(ncid, %s%s, \"%s\", %lu, %s);", atts[iatt].var == -1 ? "NC_GLOBAL" : vars[atts[iatt].var].lname, atts[iatt].var == -1 ? "" : "_id", @@ -409,7 +409,7 @@ gen_c( else { /* vector attribute */ for (jatt = 0; jatt < atts[iatt].len ; jatt++) { val_string = cstring(atts[iatt].type,atts[iatt].val,jatt); - sprintf(stmnt, " %s_%s[%d] = %s;", + snprintf(stmnt, sizeof(stmnt), " %s_%s[%d] = %s;", atts[iatt].var == -1 ? "cdf" : vars[atts[iatt].var].lname, atts[iatt].lname, jatt, @@ -418,7 +418,7 @@ gen_c( free (val_string); } - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_put_att_%s(ncid, %s%s, \"%s\", %s, %lu, %s_%s);", ncatype(atts[iatt].type), atts[iatt].var == -1 ? "NC_GLOBAL" : vars[atts[iatt].var].lname, @@ -610,21 +610,21 @@ gen_fortran( if (ndims > 0) { fline("* dimension ids"); for (idim = 0; idim < ndims; idim++) { - sprintf(stmnt, "integer %s_dim", dims[idim].lname); + snprintf(stmnt, sizeof(stmnt), "integer %s_dim", dims[idim].lname); fline(stmnt); } fline("* dimension lengths"); for (idim = 0; idim < ndims; idim++) { - sprintf(stmnt, "integer %s_len", dims[idim].lname); + snprintf(stmnt, sizeof(stmnt), "integer %s_len", dims[idim].lname); fline(stmnt); } for (idim = 0; idim < ndims; idim++) { if (dims[idim].size == NC_UNLIMITED) { - sprintf(stmnt, "parameter (%s_len = NF_UNLIMITED)", + snprintf(stmnt, sizeof(stmnt), "parameter (%s_len = NF_UNLIMITED)", dims[idim].lname); } else { - sprintf(stmnt, "parameter (%s_len = %lu)", + snprintf(stmnt, sizeof(stmnt), "parameter (%s_len = %lu)", dims[idim].lname, (unsigned long) dims[idim].size); } @@ -641,17 +641,17 @@ gen_fortran( if (nvars > 0) { fline("* variable ids"); for (ivar = 0; ivar < nvars; ivar++) { - sprintf(stmnt, "integer %s_id", vars[ivar].lname); + snprintf(stmnt, sizeof(stmnt), "integer %s_id", vars[ivar].lname); fline(stmnt); } fline("* rank (number of dimensions) for each variable"); for (ivar = 0; ivar < nvars; ivar++) { - sprintf(stmnt, "integer %s_rank", vars[ivar].lname); + snprintf(stmnt, sizeof(stmnt), "integer %s_rank", vars[ivar].lname); fline(stmnt); } for (ivar = 0; ivar < nvars; ivar++) { - sprintf(stmnt, "parameter (%s_rank = %d)", vars[ivar].lname, + snprintf(stmnt, sizeof(stmnt), "parameter (%s_rank = %d)", vars[ivar].lname, vars[ivar].ndims); fline(stmnt); } @@ -659,7 +659,7 @@ gen_fortran( fline("* variable shapes"); for (ivar = 0; ivar < nvars; ivar++) { if (vars[ivar].ndims > 0) { - sprintf(stmnt, "integer %s_dims(%s_rank)", + snprintf(stmnt, sizeof(stmnt), "integer %s_dims(%s_rank)", vars[ivar].lname, vars[ivar].lname); fline(stmnt); } @@ -683,14 +683,14 @@ gen_fortran( continue; } if (v->ndims == 0) { /* scalar */ - sprintf(stmnt, "%s %s", ncftype(v->type), + snprintf(stmnt, sizeof(stmnt), "%s %s", ncftype(v->type), v->lname); } else { - sprintf(stmnt, "%s %s(", ncftype(v->type), + snprintf(stmnt, sizeof(stmnt), "%s %s(", ncftype(v->type), v->lname); /* reverse dimensions for FORTRAN */ for (idim = v->ndims-1; idim >= 0; idim--) { - sprintf(s2, "%s_len, ", + snprintf(s2, sizeof(s2), "%s_len, ", dims[v->dims[idim]].lname); strcat(stmnt, s2); } @@ -719,7 +719,7 @@ gen_fortran( fline("* attribute vectors"); for (itype = 0; itype < ntypes; itype++) { if (types[itype] != NC_CHAR && max_atts[(int)types[itype]] > 0) { - sprintf(stmnt, "%s %sval(%lu)", ncftype(types[itype]), + snprintf(stmnt, sizeof(stmnt), "%s %sval(%lu)", ncftype(types[itype]), nfstype(types[itype]), (unsigned long) max_atts[(int)types[itype]]); fline(stmnt); @@ -730,14 +730,14 @@ gen_fortran( /* create netCDF file, uses NC_CLOBBER mode */ fline("* enter define mode"); if (!cmode_modifier) { - sprintf(stmnt, "iret = nf_create(\'%s\', NF_CLOBBER, ncid)", filename); + snprintf(stmnt, sizeof(stmnt), "iret = nf_create(\'%s\', NF_CLOBBER, ncid)", filename); } else if (cmode_modifier & NC_64BIT_OFFSET) { - sprintf(stmnt, "iret = nf_create(\'%s\', OR(NF_CLOBBER,NF_64BIT_OFFSET), ncid)", filename); + snprintf(stmnt, sizeof(stmnt), "iret = nf_create(\'%s\', OR(NF_CLOBBER,NF_64BIT_OFFSET), ncid)", filename); #ifdef USE_NETCDF4 } else if (cmode_modifier & NC_CLASSIC_MODEL) { - sprintf(stmnt, "iret = nf_create(\'%s\', OR(NF_CLOBBER,NC_NETCDF4,NC_CLASSIC_MODEL), ncid)", filename); + snprintf(stmnt, sizeof(stmnt), "iret = nf_create(\'%s\', OR(NF_CLOBBER,NC_NETCDF4,NC_CLASSIC_MODEL), ncid)", filename); } else if (cmode_modifier & NC_NETCDF4) { - sprintf(stmnt, "iret = nf_create(\'%s\', OR(NF_CLOBBER,NF_NETCDF4), ncid)", filename); + snprintf(stmnt, sizeof(stmnt), "iret = nf_create(\'%s\', OR(NF_CLOBBER,NF_NETCDF4), ncid)", filename); #endif } else { derror("unknown cmode modifier"); @@ -750,10 +750,10 @@ gen_fortran( fline("* define dimensions"); for (idim = 0; idim < ndims; idim++) { if (dims[idim].size == NC_UNLIMITED) - sprintf(stmnt, "iret = nf_def_dim(ncid, \'%s\', NF_UNLIMITED, %s_dim)", + snprintf(stmnt, sizeof(stmnt), "iret = nf_def_dim(ncid, \'%s\', NF_UNLIMITED, %s_dim)", dims[idim].name, dims[idim].lname); else - sprintf(stmnt, "iret = nf_def_dim(ncid, \'%s\', %lu, %s_dim)", + snprintf(stmnt, sizeof(stmnt), "iret = nf_def_dim(ncid, \'%s\', %lu, %s_dim)", dims[idim].name, (unsigned long) dims[idim].size, dims[idim].lname); fline(stmnt); @@ -765,14 +765,14 @@ gen_fortran( fline("* define variables"); for (ivar = 0; ivar < nvars; ivar++) { for (idim = 0; idim < vars[ivar].ndims; idim++) { - sprintf(stmnt, "%s_dims(%d) = %s_dim", + snprintf(stmnt, sizeof(stmnt), "%s_dims(%d) = %s_dim", vars[ivar].lname, vars[ivar].ndims - idim, /* reverse dimensions */ dims[vars[ivar].dims[idim]].lname); fline(stmnt); } if (vars[ivar].ndims > 0) { /* a dimensioned variable */ - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), "iret = nf_def_var(ncid, \'%s\', %s, %s_rank, %s_dims, %s_id)", vars[ivar].name, ftypename(vars[ivar].type), @@ -780,7 +780,7 @@ gen_fortran( vars[ivar].lname, vars[ivar].lname); } else { /* a scalar */ - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), "iret = nf_def_var(ncid, \'%s\', %s, %s_rank, 0, %s_id)", vars[ivar].name, ftypename(vars[ivar].type), @@ -798,7 +798,7 @@ gen_fortran( for (iatt = 0; iatt < natts; iatt++) { if (atts[iatt].type == NC_CHAR) { /* string */ val_string = fstrstr((char *) atts[iatt].val, atts[iatt].len); - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), "iret = nf_put_att_text(ncid, %s%s, \'%s\', %lu, %s)", atts[iatt].var == -1 ? "NF_GLOBAL" : vars[atts[iatt].var].lname, atts[iatt].var == -1 ? "" : "_id", @@ -811,7 +811,7 @@ gen_fortran( } else { for (jatt = 0; jatt < atts[iatt].len ; jatt++) { val_string = fstring(atts[iatt].type,atts[iatt].val,jatt); - sprintf(stmnt, "%sval(%d) = %s", + snprintf(stmnt, sizeof(stmnt), "%sval(%d) = %s", nfstype(atts[iatt].type), jatt+1, val_string); @@ -819,7 +819,7 @@ gen_fortran( free (val_string); } - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), "iret = nf_put_att_%s(ncid, %s%s, \'%s\', %s, %lu, %sval)", nfftype(atts[iatt].type), atts[iatt].var == -1 ? "NCGLOBAL" : vars[atts[iatt].var].lname, @@ -1206,7 +1206,7 @@ fstrstr( *cp = '\0'; was_print = 1; } else { - sprintf(tstr, "char(%d)", (unsigned char)*istr); + snprintf(tstr, sizeof(tstr), "char(%d)", (unsigned char)*istr); strcat(cp, tstr); cp += strlen(tstr); was_print = 0; @@ -1239,7 +1239,7 @@ fstrstr( *cp++ = '\''; *cp = '\0'; } - sprintf(tstr, "//char(%d)", (unsigned char)*istr); + snprintf(tstr, sizeof(tstr), "//char(%d)", (unsigned char)*istr); strcat(cp, tstr); cp += strlen(tstr); was_print = 0; @@ -1346,13 +1346,13 @@ cl_fortran(void) if (have_rec_var) { fline(" "); fline("* Write record variables"); - sprintf(stmnt, "call writerecs(ncid,"); + snprintf(stmnt, sizeof(stmnt), "call writerecs(ncid,"); /* generate parameter list for subroutine to write record vars */ for (ivar = 0; ivar < nvars; ivar++) { struct vars *v = &vars[ivar]; /* if a record variable, include id in parameter list */ if (v->ndims > 0 && v->dims[0] == rec_dim) { - sprintf(s2, "%s_id,", v->lname); + snprintf(s2, sizeof(s2), "%s_id,", v->lname); strcat(stmnt, s2); } } @@ -1372,11 +1372,11 @@ cl_fortran(void) fline(" "); if (have_rec_var) { - sprintf(stmnt, "subroutine writerecs(ncid,"); + snprintf(stmnt, sizeof(stmnt), "subroutine writerecs(ncid,"); for (ivar = 0; ivar < nvars; ivar++) { struct vars *v = &vars[ivar]; if (v->ndims > 0 && v->dims[0] == rec_dim) { - sprintf(s2, "%s_id,", v->lname); + snprintf(s2, sizeof(s2), "%s_id,", v->lname); strcat(stmnt, s2); } } @@ -1394,7 +1394,7 @@ cl_fortran(void) for (ivar = 0; ivar < nvars; ivar++) { struct vars *v = &vars[ivar]; if (v->ndims > 0 && v->dims[0] == rec_dim) { - sprintf(stmnt, "integer %s_id", v->lname); + snprintf(stmnt, sizeof(stmnt), "integer %s_id", v->lname); fline(stmnt); } } @@ -1413,9 +1413,9 @@ cl_fortran(void) for (idim = 0; idim < ndims; idim++) { /* if used in a record variable and not record dimension */ if (used_in_rec_var(idim) && dims[idim].size != NC_UNLIMITED) { - sprintf(stmnt, "integer %s_len", dims[idim].lname); + snprintf(stmnt, sizeof(stmnt), "integer %s_len", dims[idim].lname); fline(stmnt); - sprintf(stmnt, "parameter (%s_len = %lu)", + snprintf(stmnt, sizeof(stmnt), "parameter (%s_len = %lu)", dims[idim].lname, (unsigned long) dims[idim].size); fline(stmnt); } @@ -1426,14 +1426,14 @@ cl_fortran(void) for (ivar = 0; ivar < nvars; ivar++) { struct vars *v = &vars[ivar]; if (v->ndims > 0 && v->dims[0] == rec_dim) { - sprintf(stmnt, "integer %s_rank", v->lname); + snprintf(stmnt, sizeof(stmnt), "integer %s_rank", v->lname); fline(stmnt); } } for (ivar = 0; ivar < nvars; ivar++) { struct vars *v = &vars[ivar]; if (v->ndims > 0 && v->dims[0] == rec_dim) { - sprintf(stmnt, "parameter (%s_rank = %d)", v->lname, + snprintf(stmnt, sizeof(stmnt), "parameter (%s_rank = %d)", v->lname, v->ndims); fline(stmnt); } @@ -1443,7 +1443,7 @@ cl_fortran(void) for (ivar = 0; ivar < nvars; ivar++) { struct vars *v = &vars[ivar]; if (v->ndims > 0 && v->dims[0] == rec_dim) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), "integer %s_start(%s_rank), %s_count(%s_rank)", v->lname, v->lname, v->lname, v->lname); fline(stmnt); @@ -1459,25 +1459,25 @@ cl_fortran(void) char *sp; fline(" "); - sprintf(stmnt, "integer %s_nr", v->lname); + snprintf(stmnt, sizeof(stmnt), "integer %s_nr", v->lname); fline(stmnt); if (v->nrecs > 0) { - sprintf(stmnt, "parameter (%s_nr = %lu)", + snprintf(stmnt, sizeof(stmnt), "parameter (%s_nr = %lu)", v->lname, (unsigned long) v->nrecs); } else { - sprintf(stmnt, "parameter (%s_nr = 1)", + snprintf(stmnt, sizeof(stmnt), "parameter (%s_nr = 1)", v->lname); } fline(stmnt); if (v->type != NC_CHAR) { - sprintf(stmnt, "%s %s(", ncftype(v->type), + snprintf(stmnt, sizeof(stmnt), "%s %s(", ncftype(v->type), v->lname); /* reverse dimensions for FORTRAN */ for (idim = v->ndims-1; idim >= 0; idim--) { if(v->dims[idim] == rec_dim) { - sprintf(s2, "%s_nr, ", v->lname); + snprintf(s2, sizeof(s2), "%s_nr, ", v->lname); } else { - sprintf(s2, "%s_len, ", + snprintf(s2, sizeof(s2), "%s_len, ", dims[v->dims[idim]].lname); } strcat(stmnt, s2); @@ -1507,7 +1507,7 @@ cl_fortran(void) for (idim = 1; idim < v->ndims; idim++) { rec_len *= dims[v->dims[idim]].size; } - sprintf(stmnt,"data %s /%lu * %s/", v->lname, + snprintf(stmnt, sizeof(stmnt),"data %s /%lu * %s/", v->lname, (unsigned long) rec_len, f_fill_name(v->type)); fline(stmnt); @@ -1521,28 +1521,28 @@ cl_fortran(void) if (v->ndims > 0 && v->dims[0] == rec_dim) { if (!v->has_data) continue; - sprintf(stmnt, "* store %s", v->name); + snprintf(stmnt, sizeof(stmnt), "* store %s", v->name); fline(stmnt); for (idim = 0; idim < v->ndims; idim++) { - sprintf(stmnt, "%s_start(%d) = 1", v->lname, idim+1); + snprintf(stmnt, sizeof(stmnt), "%s_start(%d) = 1", v->lname, idim+1); fline(stmnt); } for (idim = v->ndims-1; idim > 0; idim--) { - sprintf(stmnt, "%s_count(%d) = %s_len", v->lname, + snprintf(stmnt, sizeof(stmnt), "%s_count(%d) = %s_len", v->lname, v->ndims - idim, dims[v->dims[idim]].lname); fline(stmnt); } - sprintf(stmnt, "%s_count(%d) = %s_nr", v->lname, + snprintf(stmnt, sizeof(stmnt), "%s_count(%d) = %s_nr", v->lname, v->ndims, v->lname); fline(stmnt); if (v->type != NC_CHAR) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), "iret = nf_put_vara_%s(ncid, %s_id, %s_start, %s_count, %s)", nfftype(v->type), v->lname, v->lname, v->lname, v->lname); } else { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), "iret = nf_put_vara_%s(ncid, %s_id, %s_start, %s_count, %s)", nfftype(v->type), v->lname, v->lname, v->lname, v->data_stmnt); diff --git a/ncgen3/load.c b/ncgen3/load.c index 634751d664..df0db88ee0 100644 --- a/ncgen3/load.c +++ b/ncgen3/load.c @@ -77,22 +77,22 @@ gen_load_c( return; cline(""); - sprintf(stmnt, " {\t\t\t/* store %s */", vars[varnum].name); + snprintf(stmnt, sizeof(stmnt), " {\t\t\t/* store %s */", vars[varnum].name); cline(stmnt); if (vars[varnum].ndims > 0) { if (vars[varnum].dims[0] == rec_dim) { - sprintf(stmnt, " static size_t %s_start[RANK_%s];", + snprintf(stmnt, sizeof(stmnt), " static size_t %s_start[RANK_%s];", vars[varnum].lname, vars[varnum].lname); cline(stmnt); - sprintf(stmnt, " static size_t %s_count[RANK_%s];", + snprintf(stmnt, sizeof(stmnt), " static size_t %s_count[RANK_%s];", vars[varnum].lname, vars[varnum].lname); cline(stmnt); } /* load variable with data values using static initialization */ - sprintf(stmnt, " static %s %s[] = {", + snprintf(stmnt, sizeof(stmnt), " static %s %s[] = {", ncctype(vars[varnum].type), vars[varnum].lname); @@ -100,7 +100,7 @@ gen_load_c( switch (vars[varnum].type) { case NC_CHAR: val_string = cstrstr((char *) rec_start, var_len); - sprintf(s2, "%s", val_string); + snprintf(s2, sizeof(s2), "%s", val_string); strlcat(stmnt, s2, C_MAX_STMNT); free(val_string); break; @@ -127,23 +127,23 @@ gen_load_c( switch (vars[varnum].type) { case NC_BYTE: assert(charvalp != NULL); - sprintf(s2, "%d, ", *charvalp++); + snprintf(s2, sizeof(s2), "%d, ", *charvalp++); break; case NC_SHORT: assert(shortvalp != NULL); - sprintf(s2, "%d, ", *shortvalp++); + snprintf(s2, sizeof(s2), "%d, ", *shortvalp++); break; case NC_INT: assert(intvalp != NULL); - sprintf(s2, "%ld, ", (long)*intvalp++); + snprintf(s2, sizeof(s2), "%ld, ", (long)*intvalp++); break; case NC_FLOAT: assert(floatvalp != NULL); - sprintf(s2, "%.8g, ", *floatvalp++); + snprintf(s2, sizeof(s2), "%.8g, ", *floatvalp++); break; case NC_DOUBLE: assert(doublevalp != NULL); - sprintf(s2, "%#.16g", *doublevalp++); + snprintf(s2, sizeof(s2), "%#.16g", *doublevalp++); tztrim(s2); strcat(s2, ", "); break; @@ -162,23 +162,23 @@ gen_load_c( switch (vars[varnum].type) { case NC_BYTE: assert(charvalp != NULL); - sprintf(s2, "%d", *charvalp); + snprintf(s2, sizeof(s2), "%d", *charvalp); break; case NC_SHORT: assert(shortvalp != NULL); - sprintf(s2, "%d", *shortvalp); + snprintf(s2, sizeof(s2), "%d", *shortvalp); break; case NC_INT: assert(intvalp != NULL); - sprintf(s2, "%ld", (long)*intvalp); + snprintf(s2, sizeof(s2), "%ld", (long)*intvalp); break; case NC_FLOAT: assert(floatvalp != NULL); - sprintf(s2, "%.8g", *floatvalp); + snprintf(s2, sizeof(s2), "%.8g", *floatvalp); break; case NC_DOUBLE: assert(doublevalp != NULL); - sprintf(s2, "%#.16g", *doublevalp++); + snprintf(s2, sizeof(s2), "%#.16g", *doublevalp++); tztrim(s2); break; default: break; @@ -198,7 +198,7 @@ gen_load_c( cline(stmnt); if (vars[varnum].dims[0] == rec_dim) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " %s_len = %lu; /* number of records of %s data */", dims[rec_dim].lname, (unsigned long)vars[varnum].nrecs, /* number of recs for this variable */ @@ -206,14 +206,14 @@ gen_load_c( cline(stmnt); for (idim = 0; idim < vars[varnum].ndims; idim++) { - sprintf(stmnt, " %s_start[%d] = 0;", + snprintf(stmnt, sizeof(stmnt), " %s_start[%d] = 0;", vars[varnum].lname, idim); cline(stmnt); } for (idim = 0; idim < vars[varnum].ndims; idim++) { - sprintf(stmnt, " %s_count[%d] = %s_len;", + snprintf(stmnt, sizeof(stmnt), " %s_count[%d] = %s_len;", vars[varnum].lname, idim, dims[vars[varnum].dims[idim]].lname); @@ -222,7 +222,7 @@ gen_load_c( } if (vars[varnum].dims[0] == rec_dim) { - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_put_vara_%s(ncid, %s_id, %s_start, %s_count, %s);", ncstype(vars[varnum].type), vars[varnum].lname, @@ -230,7 +230,7 @@ gen_load_c( vars[varnum].lname, vars[varnum].lname); } else { /* non-record variables */ - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_put_var_%s(ncid, %s_id, %s);", ncstype(vars[varnum].type), vars[varnum].lname, @@ -239,7 +239,7 @@ gen_load_c( cline(stmnt); } else { /* scalar variables */ /* load variable with data values using static initialization */ - sprintf(stmnt, " static %s %s = ", + snprintf(stmnt, sizeof(stmnt), " static %s %s = ", ncctype(vars[varnum].type), vars[varnum].lname); @@ -247,28 +247,28 @@ gen_load_c( case NC_CHAR: val_string = cstrstr((char *) rec_start, var_len); val_string[strlen(val_string)-1] = '\0'; - sprintf(s2, "'%s'", &val_string[1]); + snprintf(s2, sizeof(s2), "'%s'", &val_string[1]); free(val_string); break; case NC_BYTE: charvalp = (char *) rec_start; - sprintf(s2, "%d", *charvalp); + snprintf(s2, sizeof(s2), "%d", *charvalp); break; case NC_SHORT: shortvalp = (short *) rec_start; - sprintf(s2, "%d", *shortvalp); + snprintf(s2, sizeof(s2), "%d", *shortvalp); break; case NC_INT: intvalp = (int *) rec_start; - sprintf(s2, "%ld", (long)*intvalp); + snprintf(s2, sizeof(s2), "%ld", (long)*intvalp); break; case NC_FLOAT: floatvalp = (float *) rec_start; - sprintf(s2, "%.8g", *floatvalp); + snprintf(s2, sizeof(s2), "%.8g", *floatvalp); break; case NC_DOUBLE: doublevalp = (double *) rec_start; - sprintf(s2, "%#.16g", *doublevalp++); + snprintf(s2, sizeof(s2), "%#.16g", *doublevalp++); tztrim(s2); break; default: break; @@ -276,7 +276,7 @@ gen_load_c( strlcat(stmnt, s2, C_MAX_STMNT); strlcat(stmnt,";", C_MAX_STMNT); cline(stmnt); - sprintf(stmnt, + snprintf(stmnt, sizeof(stmnt), " stat = nc_put_var_%s(ncid, %s_id, &%s);", ncstype(vars[varnum].type), vars[varnum].lname, @@ -345,14 +345,14 @@ f_var_init( int ival; /* load variable with data values */ - sprintf(stmnt, "data %s /",vars[varnum].lname); + snprintf(stmnt, sizeof(stmnt), "data %s /",vars[varnum].lname); stmnt_len = strlen(stmnt); switch (vars[varnum].type) { case NC_BYTE: charvalp = (char *) rec_start; for (ival = 0; ival < var_len-1; ival++) { val_string = fstring(NC_BYTE,(void *)charvalp++,0); - sprintf(s2, "%s, ", val_string); + snprintf(s2, sizeof(s2), "%s, ", val_string); fstrcat(stmnt, s2, &stmnt_len); free(val_string); } @@ -363,40 +363,40 @@ f_var_init( case NC_SHORT: shortvalp = (short *) rec_start; for (ival = 0; ival < var_len-1; ival++) { - sprintf(s2, "%d, ", *shortvalp++); + snprintf(s2, sizeof(s2), "%d, ", *shortvalp++); fstrcat(stmnt, s2, &stmnt_len); } - sprintf(s2, "%d", *shortvalp); + snprintf(s2, sizeof(s2), "%d", *shortvalp); fstrcat(stmnt, s2, &stmnt_len); break; case NC_INT: intvalp = (int *) rec_start; for (ival = 0; ival < var_len-1; ival++) { - sprintf(s2, "%ld, ", (long)*intvalp++); + snprintf(s2, sizeof(s2), "%ld, ", (long)*intvalp++); fstrcat(stmnt, s2, &stmnt_len); } - sprintf(s2, "%ld", (long)*intvalp); + snprintf(s2, sizeof(s2), "%ld", (long)*intvalp); fstrcat(stmnt, s2, &stmnt_len); break; case NC_FLOAT: floatvalp = (float *) rec_start; for (ival = 0; ival < var_len-1; ival++) { - sprintf(s2, "%.8g, ", *floatvalp++); + snprintf(s2, sizeof(s2), "%.8g, ", *floatvalp++); fstrcat(stmnt, s2, &stmnt_len); } - sprintf(s2, "%.8g", *floatvalp); + snprintf(s2, sizeof(s2), "%.8g", *floatvalp); fstrcat(stmnt, s2, &stmnt_len); break; case NC_DOUBLE: doublevalp = (double *) rec_start; for (ival = 0; ival < var_len-1; ival++) { - sprintf(s2, "%#.16g", *doublevalp++); + snprintf(s2, sizeof(s2), "%#.16g", *doublevalp++); tztrim(s2); expe2d(s2); /* change 'e' to 'd' in exponent */ fstrcat(s2, ", ", &stmnt_len); fstrcat(stmnt, s2, &stmnt_len); } - sprintf(s2, "%#.16g", *doublevalp++); + snprintf(s2, sizeof(s2), "%#.16g", *doublevalp++); tztrim(s2); expe2d(s2); fstrcat(stmnt, s2, &stmnt_len); @@ -432,7 +432,7 @@ gen_load_fortran( return; if (v->ndims == 0 || v->dims[0] != rec_dim) { - sprintf(stmnt, "* store %s", v->name); + snprintf(stmnt, sizeof(stmnt), "* store %s", v->name); fline(stmnt); } @@ -447,7 +447,7 @@ gen_load_fortran( return; } if (v->type != NC_CHAR) { - sprintf(stmnt, "iret = nf_put_var_%s(ncid, %s_id, %s)", + snprintf(stmnt, sizeof(stmnt), "iret = nf_put_var_%s(ncid, %s_id, %s)", nfftype(v->type), v->lname, v->lname); } else { char *char_expr = fstrstr(rec_start, valnum); @@ -459,7 +459,7 @@ gen_load_fortran( v->lname); exit(9); } - sprintf(stmnt, "iret = nf_put_var_%s(ncid, %s_id, %s)", + snprintf(stmnt, sizeof(stmnt), "iret = nf_put_var_%s(ncid, %s_id, %s)", nfftype(v->type), v->lname, char_expr); free(char_expr); } diff --git a/ncgen3/ncgen.l b/ncgen3/ncgen.l index 42490f72bd..9c304f4038 100644 --- a/ncgen3/ncgen.l +++ b/ncgen3/ncgen.l @@ -181,12 +181,12 @@ FloatInf|-?Inff { /* missing value (pre-2.4 backward compatibility) */ [+-]?[0-9]*[0-9][Bb] { int ii; if (sscanf((char*)yytext, "%d", &ii) != 1) { - sprintf(errstr,"bad byte constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad byte constant: %s",(char*)yytext); yyerror(errstr); } byte_val = ii; if (ii != (int)byte_val) { - sprintf(errstr,"byte constant out of range (-128,127): %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"byte constant out of range (-128,127): %s",(char*)yytext); yyerror(errstr); } return (BYTE_CONST); @@ -194,14 +194,14 @@ FloatInf|-?Inff { /* missing value (pre-2.4 backward compatibility) */ [+-]?[0-9]*\.[0-9]*{exp}?[LlDd]?|[+-]?[0-9]*{exp}[LlDd]? { if (sscanf((char*)yytext, "%le", &double_val) != 1) { - sprintf(errstr,"bad long or double constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad long or double constant: %s",(char*)yytext); yyerror(errstr); } return (DOUBLE_CONST); } [+-]?[0-9]*\.[0-9]*{exp}?[Ff]|[+-]?[0-9]*{exp}[Ff] { if (sscanf((char*)yytext, "%e", &float_val) != 1) { - sprintf(errstr,"bad float constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad float constant: %s",(char*)yytext); yyerror(errstr); } return (FLOAT_CONST); @@ -209,7 +209,7 @@ FloatInf|-?Inff { /* missing value (pre-2.4 backward compatibility) */ [+-]?[0-9]+[sS]|0[xX][0-9a-fA-F]+[sS] { int tmp = 0; if (sscanf((char*)yytext, "%d", &tmp) != 1) { - sprintf(errstr,"bad short constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad short constant: %s",(char*)yytext); yyerror(errstr); } short_val = (short)tmp; @@ -220,7 +220,7 @@ FloatInf|-?Inff { /* missing value (pre-2.4 backward compatibility) */ errno = 0; double_val = strtod((char*)yytext, &ptr); if (errno != 0 && double_val == 0.0) { - sprintf(errstr,"bad numerical constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad numerical constant: %s",(char*)yytext); yyerror(errstr); } if (double_val < XDR_INT_MIN ||double_val > XDR_INT_MAX) { @@ -236,7 +236,7 @@ FloatInf|-?Inff { /* missing value (pre-2.4 backward compatibility) */ errno = 0; long_val = strtol((char*)yytext, &ptr, 0); if (errno != 0) { - sprintf(errstr,"bad long constant: %s",(char*)yytext); + snprintf(errstr, sizeof(errstr),"bad long constant: %s",(char*)yytext); yyerror(errstr); } if (long_val < XDR_INT_MIN || long_val > XDR_INT_MAX) { diff --git a/ncgen3/ncgenl.c b/ncgen3/ncgenl.c index 6178f25d0c..1db8ff1d69 100644 --- a/ncgen3/ncgenl.c +++ b/ncgen3/ncgenl.c @@ -1347,12 +1347,12 @@ YY_RULE_SETUP { int ii; if (sscanf((char*)ncgtext, "%d", &ii) != 1) { - sprintf(errstr,"bad byte constant: %s",(char*)ncgtext); + snprintf(errstr,sizeof(errstr),"bad byte constant: %s",(char*)ncgtext); yyerror(errstr); } byte_val = ii; if (ii != (int)byte_val) { - sprintf(errstr,"byte constant out of range (-128,127): %s",(char*)ncgtext); + snprintf(errstr,sizeof(errstr),"byte constant out of range (-128,127): %s",(char*)ncgtext); yyerror(errstr); } return (BYTE_CONST); @@ -1363,7 +1363,7 @@ YY_RULE_SETUP #line 195 "ncgen.l" { if (sscanf((char*)ncgtext, "%le", &double_val) != 1) { - sprintf(errstr,"bad long or double constant: %s",(char*)ncgtext); + snprintf(errstr,sizeof(errstr),"bad long or double constant: %s",(char*)ncgtext); yyerror(errstr); } return (DOUBLE_CONST); @@ -1374,7 +1374,7 @@ YY_RULE_SETUP #line 202 "ncgen.l" { if (sscanf((char*)ncgtext, "%e", &float_val) != 1) { - sprintf(errstr,"bad float constant: %s",(char*)ncgtext); + snprintf(errstr,sizeof(errstr),"bad float constant: %s",(char*)ncgtext); yyerror(errstr); } return (FLOAT_CONST); @@ -1386,7 +1386,7 @@ YY_RULE_SETUP { int tmp = 0; if (sscanf((char*)ncgtext, "%d", &tmp) != 1) { - sprintf(errstr,"bad short constant: %s",(char*)ncgtext); + snprintf(errstr,sizeof(errstr),"bad short constant: %s",(char*)ncgtext); yyerror(errstr); } short_val = (short)tmp; @@ -1401,7 +1401,7 @@ YY_RULE_SETUP errno = 0; double_val = strtod((char*)ncgtext, &ptr); if (errno != 0 && double_val == 0.0) { - sprintf(errstr,"bad numerical constant: %s",(char*)ncgtext); + snprintf(errstr,sizeof(errstr),"bad numerical constant: %s",(char*)ncgtext); yyerror(errstr); } if (double_val < XDR_INT_MIN ||double_val > XDR_INT_MAX) { @@ -1421,7 +1421,7 @@ YY_RULE_SETUP errno = 0; long_val = strtol((char*)ncgtext, &ptr, 0); if (errno != 0) { - sprintf(errstr,"bad long constant: %s",(char*)ncgtext); + snprintf(errstr,sizeof(errstr),"bad long constant: %s",(char*)ncgtext); yyerror(errstr); } if (long_val < XDR_INT_MIN || long_val > XDR_INT_MAX) { diff --git a/ncgen3/ncgeny.c b/ncgen3/ncgeny.c index 494a626429..2e87109d08 100644 --- a/ncgen3/ncgeny.c +++ b/ncgen3/ncgeny.c @@ -1082,7 +1082,7 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, return 1; } - /* Avoid sprintf, as that infringes on the user's name space. + /* Avoid snprintf, as that infringes on the user's name space. Don't have undefined behavior even if the translation produced a string with the wrong number of "%s"s. */ { diff --git a/nctest/driver.c b/nctest/driver.c index 716a178092..20c68b1039 100644 --- a/nctest/driver.c +++ b/nctest/driver.c @@ -86,7 +86,7 @@ main(int argc, char **argv) continue; /* Come up with a test file name. */ - sprintf(testfile, "nctest_%s.nc", format_name[i]); + snprintf(testfile, sizeof(testfile), "nctest_%s.nc", format_name[i]); printf("Testing %s with file %s.\n", format_name[i], testfile); /* Set the default format. */ diff --git a/nczarr_test/bm_chunks3.c b/nczarr_test/bm_chunks3.c index 9a34bda331..66d0b64a0d 100644 --- a/nczarr_test/bm_chunks3.c +++ b/nczarr_test/bm_chunks3.c @@ -348,10 +348,10 @@ test(Tag tag, int reading, int ncid, int varid, int rank, int index, size_t* dim count[index] = 1; if(chunks != NULL) - sprintf(time_mess,"%s %s %3ld %3ld %3ld %3ld %3ld %3ld", tagnames[tag], rwnames[reading], + snprintf(time_mess, sizeof(time_mess),"%s %s %3ld %3ld %3ld %3ld %3ld %3ld", tagnames[tag], rwnames[reading], count[0], count[1], count[2], chunks[0], chunks[1], chunks[2]); else - sprintf(time_mess,"%s %s %3ld %3ld %3ld", tagnames[tag], rwnames[reading], + snprintf(time_mess, sizeof(time_mess),"%s %s %3ld %3ld %3ld", tagnames[tag], rwnames[reading], count[0], count[1], count[2]); TIMING_START ; diff --git a/nczarr_test/test_zchunks.c b/nczarr_test/test_zchunks.c index 7307910aff..5e7cf6213a 100644 --- a/nczarr_test/test_zchunks.c +++ b/nczarr_test/test_zchunks.c @@ -110,7 +110,7 @@ main(int argc, char **argv) for (d = 0; d < NUM_DIM; d++) { - sprintf(dim_name, "dim_%d", dim_len[d]); + snprintf(dim_name, sizeof(dim_name), "dim_%d", dim_len[d]); #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE printf("creating dim[%d] %s = %d\n", d, dim_name, dim_len[d]); #endif @@ -119,7 +119,7 @@ main(int argc, char **argv) for (t = 0; t < NUM_TYPE; t++) { - sprintf(var_name, "var_%d", type_id[t]); + snprintf(var_name, sizeof(var_name), "var_%d", type_id[t]); if (nc_def_var(ncid, var_name, type_id[t], NUM_DIM, dimid, &varid[t])) ERR; if (nc_inq_var_chunking(ncid, varid[t], &storage, chunksize_in)) ERR; #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE @@ -142,7 +142,7 @@ main(int argc, char **argv) for (t = 0; t < NUM_TYPE; t++) { - sprintf(var_name, "var_%d", type_id[t]); + snprintf(var_name, sizeof(var_name), "var_%d", type_id[t]); if (nc_inq_var_chunking(ncid, varid[t], &storage, chunksize_in)) ERR; if (storage) ERR; #ifdef PRINT_DEFAULT_CHUNKSIZE_TABLE @@ -242,7 +242,7 @@ main(int argc, char **argv) /* Oh that tricky Cardinal Richelieu, he had many plans! */ for (i = 0; i < NUM_PLANS; i++) { - sprintf(plan_name, "Richelieu_sneaky_plan_%d", i); + snprintf(plan_name, sizeof(plan_name), "Richelieu_sneaky_plan_%d", i); if (nc_def_var(ncid, plan_name, i % (NC_STRING - 1) + 1, NDIMS_3, dimids, &varid[i])) ERR; if (nc_def_var_chunking(ncid, varid[i], 0, chunksize)) ERR; diff --git a/nczarr_test/test_zchunks2.c b/nczarr_test/test_zchunks2.c index 54963761c8..271756d138 100644 --- a/nczarr_test/test_zchunks2.c +++ b/nczarr_test/test_zchunks2.c @@ -165,7 +165,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -203,7 +203,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -241,7 +241,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -282,7 +282,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -322,7 +322,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -364,7 +364,7 @@ main(int argc, char **argv) for (d = 0; d < NDIMS3; d++) { dim_len[d] = rand(); - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -403,7 +403,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } @@ -442,7 +442,7 @@ main(int argc, char **argv) /* Create a few dimensions. */ for (d = 0; d < NDIMS3; d++) { - sprintf(dim_name, "dim_%d", d); + snprintf(dim_name, sizeof(dim_name), "dim_%d", d); if (nc_def_dim(ncid, dim_name, dim_len[d], &dimids[d])) ERR; } diff --git a/oc2/dapparse.c b/oc2/dapparse.c index 8d039ad557..9a4517f794 100644 --- a/oc2/dapparse.c +++ b/oc2/dapparse.c @@ -104,7 +104,7 @@ dap_unrecognizedresponse(DAPparsestate* state) int i; char iv[32]; (void)sscanf(state->lexstate->input,"%u ",&httperr); - sprintf(iv,"%u",httperr); + snprintf(iv,sizeof(iv),"%u",httperr); state->lexstate->next = state->lexstate->input; /* Limit the amount of input to prevent runaway */ for(i=0;i<4096;i++) {if(state->lexstate->input[i] == '\0') break;} @@ -256,7 +256,7 @@ char* dimnameanon(char* basename, unsigned int index) { char name[64]; - sprintf(name,"%s_%d",basename,index); + snprintf(name,sizeof(name),"%s_%d",basename,index); return strdup(name); } diff --git a/oc2/dapy.c b/oc2/dapy.c index fc7c45d1f2..c9883f9802 100644 --- a/oc2/dapy.c +++ b/oc2/dapy.c @@ -1110,7 +1110,7 @@ yysyntax_error (YYSIZE_T *yymsg_alloc, char **yymsg, return 1; } - /* Avoid sprintf, as that infringes on the user's name space. + /* Avoid snprintf, as that infringes on the user's name space. Don't have undefined behavior even if the translation produced a string with the wrong number of "%s"s. */ { diff --git a/oc2/ocdump.c b/oc2/ocdump.c index cde4df3fd3..a87dca9722 100644 --- a/oc2/ocdump.c +++ b/oc2/ocdump.c @@ -259,27 +259,27 @@ dumpfield(size_t index, char* n8, int isxdr) line[0] = '\0'; /* offset */ - sprintf(tmp,"%6zd",index); + snprintf(tmp,sizeof(tmp),"%6zd",index); addfield(tmp,sizeof(line),line,5); memcpy(form.cv,n8,4); /* straight hex*/ - sprintf(tmp,"%08x",form.uv); + snprintf(tmp,sizeof(tmp),"%08x",form.uv); addfield(tmp,sizeof(line),line,8); if(isxdr) {swapinline32(&form.uv);} /* unsigned integer */ - sprintf(tmp,"%12u",form.uv); + snprintf(tmp,sizeof(tmp),"%12u",form.uv); addfield(tmp,sizeof(line),line,12); /* signed integer */ - sprintf(tmp,"%12d",form.sv); + snprintf(tmp,sizeof(tmp),"%12d",form.sv); addfield(tmp,sizeof(line),line,12); /* float */ - sprintf(tmp,"%#g",form.fv); + snprintf(tmp,sizeof(tmp),"%#g",form.fv); addfield(tmp,sizeof(line),line,12); /* char[4] */ @@ -303,7 +303,7 @@ dumpfield(size_t index, char* n8, int isxdr) /* double */ memcpy(dform.cv,n8,(size_t)(2*XDRUNIT)); if(isxdr) xxdrntohdouble(dform.cv,&dform.d); - sprintf(tmp,"%#g",dform.d); + snprintf(tmp,sizeof(tmp),"%#g",dform.d); addfield(tmp,sizeof(line),line,12); fprintf(stdout,"%s\n",line); @@ -378,11 +378,11 @@ simplememorydump(char* memory, size_t len, int fromxdr) unsigned int v = vx; if(!xxdr_network_order) swapinline32(&v); line[0] = '\0'; - sprintf(tmp,"%6d",i); + snprintf(tmp,sizeof(tmp),"%6d",i); addfield(tmp,sizeof(line),line,6); - sprintf(tmp,"%08x",vx); + snprintf(tmp,sizeof(tmp),"%08x",vx); addfield(tmp,sizeof(line),line,9); - sprintf(tmp,"%08x",v); + snprintf(tmp,sizeof(tmp),"%08x",v); addfield(tmp,sizeof(line),line,10); fprintf(stdout,"%s\n",line); } From adc4dc143514fb498cfd92c74fc00e18ada74c16 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Sun, 30 Apr 2023 18:36:39 -0400 Subject: [PATCH 69/77] Replaced some sprintf with snprintf with aid of new variable containing size One case required slightly complicated accounting of how much space is left in the buffer. --- libhdf5/hdf5var.c | 7 ++--- libnczarr/zvar.c | 7 ++--- nc_test/tst_def_var_fill.c | 5 ++-- ncdump/dumplib.c | 6 +++-- ncgen/util.c | 10 +++++--- ncgen3/genlib.c | 52 +++++++++++++++++++++++--------------- 6 files changed, 53 insertions(+), 34 deletions(-) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 6d37fe9099..6c2303ec45 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -121,11 +121,12 @@ give_var_secret_name(NC_VAR_INFO_T *var, const char *name) * clash. */ if (strlen(name) + strlen(NON_COORD_PREPEND) > NC_MAX_NAME) return NC_EMAXNAME; - if (!(var->alt_name = malloc((strlen(NON_COORD_PREPEND) + - strlen(name) + 1) * sizeof(char)))) + size_t alt_name_size = (strlen(NON_COORD_PREPEND) + strlen(name) + 1) * + sizeof(char); + if (!(var->alt_name = malloc(alt_name_size))) return NC_ENOMEM; - sprintf(var->alt_name, "%s%s", NON_COORD_PREPEND, name); + snprintf(var->alt_name, alt_name_size, "%s%s", NON_COORD_PREPEND, name); return NC_NOERR; } diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 27e30a9f19..5076ff740a 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -239,11 +239,12 @@ give_var_secret_name(NC_VAR_INFO_T *var, const char *name) * clash. */ if (strlen(name) + strlen(NON_COORD_PREPEND) > NC_MAX_NAME) return NC_EMAXNAME; - if (!(var->ncz_name = malloc((strlen(NON_COORD_PREPEND) + - strlen(name) + 1) * sizeof(char)))) + size_t ncz_name_size = (strlen(NON_COORD_PREPEND) + strlen(name) + 1) * + sizeof(char); + if (!(var->ncz_name = malloc(ncz_name_size))) return NC_ENOMEM; - sprintf(var->ncz_name, "%s%s", NON_COORD_PREPEND, name); + snprintf(var->ncz_name, ncz_name_size, "%s%s", NON_COORD_PREPEND, name); return NC_NOERR; } diff --git a/nc_test/tst_def_var_fill.c b/nc_test/tst_def_var_fill.c index 5c32485420..425d563c8a 100644 --- a/nc_test/tst_def_var_fill.c +++ b/nc_test/tst_def_var_fill.c @@ -46,8 +46,9 @@ int main(int argc, char** argv) { if (argc == 2) snprintf(filename, 256, "%s", argv[1]); else strcpy(filename, "tst_def_var_fill.nc"); - char *cmd_str = (char*)malloc(strlen(argv[0]) + 256); - sprintf(cmd_str, "*** TESTING C %s for def_var_fill ", argv[0]); + size_t cmd_str_len = strlen(argv[0]) + 256; + char *cmd_str = (char*)malloc(cmd_str_len); + snprintf(cmd_str, cmd_str_len, "*** TESTING C %s for def_var_fill ", argv[0]); printf("%-66s ------ ", cmd_str); fflush(stdout); free(cmd_str); diff --git a/ncdump/dumplib.c b/ncdump/dumplib.c index 03fe3b278e..b157937d16 100644 --- a/ncdump/dumplib.c +++ b/ncdump/dumplib.c @@ -1034,7 +1034,8 @@ chars_tostring( { long iel; const char *sp; - char *sout = (char *)emalloc(4*len + 5); /* max len of string */ + size_t sout_size = 4*len + 5; /* max len of string */ + char *sout = (char *)emalloc(sout_size); char *cp = sout; *cp++ = '"'; @@ -1061,7 +1062,8 @@ chars_tostring( if (isprint(uc)) *cp++ = *(char *)&uc; /* just copy, even if char is signed */ else { - sprintf(cp,"\\%.3o",uc); + size_t remaining = sout_size - (cp - sout); + snprintf(cp,remaining,"\\%.3o",uc); cp += 4; } break; diff --git a/ncgen/util.c b/ncgen/util.c index 108fff0440..589582c611 100644 --- a/ncgen/util.c +++ b/ncgen/util.c @@ -174,8 +174,9 @@ nctypename(nc_type nctype) return nctypenamesextend[(nctype - NC_GRP)]; if(nctype == NC_FILLVALUE) return "NC_FILL"; if(nctype == NC_NIL) return "NC_NIL"; - s = poolalloc(128); - sprintf(s,"NC_<%d>",nctype); + const size_t s_size = 128; + s = poolalloc(s_size); + snprintf(s,s_size,"NC_<%d>",nctype); return s; } @@ -195,8 +196,9 @@ ncclassname(nc_class ncc) if(ncc == NC_FILLVALUE) return "NC_FILL"; if(ncc >= NC_GRP && ncc <= NC_PRIM) return ncclassnames[ncc - NC_GRP]; - s = poolalloc(128); - sprintf(s,"NC_<%d>",ncc); + const size_t s_size = 128; + s = poolalloc(s_size); + snprintf(s,s_size,"NC_<%d>",ncc); return s; } diff --git a/ncgen3/genlib.c b/ncgen3/genlib.c index ead98b08df..9c9f987c6f 100644 --- a/ncgen3/genlib.c +++ b/ncgen3/genlib.c @@ -129,6 +129,7 @@ cstring( int *intp; float *floatp; double *doublep; + size_t cp_size; switch (type) { case NC_CHAR: @@ -162,34 +163,39 @@ cstring( return sp; case NC_BYTE: - cp = (char *) emalloc (7); + cp_size = 7; + cp = (char *) emalloc (cp_size); bytep = (signed char *)valp; /* Need to convert '\377' to -1, for example, on all platforms */ - (void) sprintf(cp,"%d", (signed char) *(bytep+num)); + (void) snprintf(cp,cp_size,"%d", (signed char) *(bytep+num)); return cp; case NC_SHORT: - cp = (char *) emalloc (10); + cp_size = 10; + cp = (char *) emalloc (cp_size); shortp = (short *)valp; - (void) sprintf(cp,"%d",* (shortp + num)); + (void) snprintf(cp,cp_size,"%d",* (shortp + num)); return cp; case NC_INT: - cp = (char *) emalloc (20); + cp_size = 20; + cp = (char *) emalloc (cp_size); intp = (int *)valp; - (void) sprintf(cp,"%d",* (intp + num)); + (void) snprintf(cp,cp_size,"%d",* (intp + num)); return cp; case NC_FLOAT: - cp = (char *) emalloc (20); + cp_size = 20; + cp = (char *) emalloc (cp_size); floatp = (float *)valp; - (void) sprintf(cp,"%.8g",* (floatp + num)); + (void) snprintf(cp,cp_size,"%.8g",* (floatp + num)); return cp; case NC_DOUBLE: - cp = (char *) emalloc (20); + cp_size = 20; + cp = (char *) emalloc (cp_size); doublep = (double *)valp; - (void) sprintf(cp,"%.16g",* (doublep + num)); + (void) snprintf(cp,cp_size,"%.16g",* (doublep + num)); return cp; default: @@ -1046,36 +1052,42 @@ fstring( int *intp; float *floatp; double *doublep; + size_t cp_size; switch (type) { case NC_BYTE: - cp = (char *) emalloc (10); + cp_size = 10; + cp = (char *) emalloc (cp_size); schp = (signed char *)valp; - sprintf(cp,"%d", schp[num]); + snprintf(cp,cp_size,"%d", schp[num]); return cp; case NC_SHORT: - cp = (char *) emalloc (10); + cp_size = 10; + cp = (char *) emalloc (cp_size); shortp = (short *)valp; - (void) sprintf(cp,"%d",* (shortp + num)); + (void) snprintf(cp,cp_size,"%d",* (shortp + num)); return cp; case NC_INT: - cp = (char *) emalloc (20); + cp_size = 20; + cp = (char *) emalloc (cp_size); intp = (int *)valp; - (void) sprintf(cp,"%d",* (intp + num)); + (void) snprintf(cp,cp_size,"%d",* (intp + num)); return cp; case NC_FLOAT: - cp = (char *) emalloc (20); + cp_size = 20; + cp = (char *) emalloc (cp_size); floatp = (float *)valp; - (void) sprintf(cp,"%.8g",* (floatp + num)); + (void) snprintf(cp,cp_size,"%.8g",* (floatp + num)); return cp; case NC_DOUBLE: - cp = (char *) emalloc (25); + cp_size = 25; + cp = (char *) emalloc (cp_size); doublep = (double *)valp; - (void) sprintf(cp,"%.16g",* (doublep + num)); + (void) snprintf(cp,cp_size,"%.16g",* (doublep + num)); expe2d(cp); /* change 'e' to 'd' in exponent */ return cp; From 4f15a9265e70cf5d8e4b43964cc15bff01358dee Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Sun, 30 Apr 2023 21:04:05 -0400 Subject: [PATCH 70/77] Removed a use of sprintf that required changing a function signature --- include/nctime.h | 4 ++-- libdispatch/nctime.c | 26 +++++++++++++------------- ncdump/dumplib.c | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/include/nctime.h b/include/nctime.h index bbab97228e..04419efe88 100644 --- a/include/nctime.h +++ b/include/nctime.h @@ -150,14 +150,14 @@ typedef struct timeinfo_t { # define MSC_NCTIME_EXTRA __declspec(dllimport) # endif -MSC_NCTIME_EXTRA extern void cdRel2Iso(cdCalenType timetype, char* relunits, int separator, double reltime, char* chartime); +MSC_NCTIME_EXTRA extern void cdRel2Iso(cdCalenType timetype, char* relunits, int separator, double reltime, char* chartime, size_t chartime_size); MSC_NCTIME_EXTRA extern void cdChar2Comp(cdCalenType timetype, char* chartime, cdCompTime* comptime); MSC_NCTIME_EXTRA extern void Cdh2e(CdTime *htime, double *etime); MSC_NCTIME_EXTRA extern void Cde2h(double etime, CdTimeType timeType, long baseYear, CdTime *htime); MSC_NCTIME_EXTRA extern int cdParseRelunits(cdCalenType timetype, char* relunits, cdUnitTime* unit, cdCompTime* base_comptime); MSC_NCTIME_EXTRA extern int cdSetErrOpts(int opts); #else -extern void cdRel2Iso(cdCalenType timetype, char* relunits, int separator, double reltime, char* chartime); +extern void cdRel2Iso(cdCalenType timetype, char* relunits, int separator, double reltime, char* chartime, size_t chartime_size); extern void cdChar2Comp(cdCalenType timetype, char* chartime, cdCompTime* comptime); extern void Cdh2e(CdTime *htime, double *etime); extern void Cde2h(double etime, CdTimeType timeType, long baseYear, CdTime *htime); diff --git a/libdispatch/nctime.c b/libdispatch/nctime.c index e265ba1f49..cdde2db00d 100644 --- a/libdispatch/nctime.c +++ b/libdispatch/nctime.c @@ -1085,7 +1085,7 @@ cdRel2Comp(cdCalenType timetype, char* relunits, double reltime, cdCompTime* com /* rkr: output as ISO 8601 strings */ static void -cdComp2Iso(cdCalenType timetype, int separator, cdCompTime comptime, char* time) +cdComp2Iso(cdCalenType timetype, int separator, cdCompTime comptime, char* time, size_t time_size) { double dtmp, sec; int ihr, imin, isec; @@ -1121,23 +1121,23 @@ cdComp2Iso(cdCalenType timetype, int separator, cdCompTime comptime, char* time) if(timetype & cdStandardCal){ switch (nskip) { case 0: /* sec != 0 && (int)sec != sec */ - sprintf(time,"%4.4ld-%2.2hd-%2.2hd%c%2.2d:%2.2d:%lf", + snprintf(time,time_size,"%4.4ld-%2.2hd-%2.2hd%c%2.2d:%2.2d:%lf", comptime.year,comptime.month,comptime.day,separator,ihr,imin,sec); break; case 1: - sprintf(time,"%4.4ld-%2.2hd-%2.2hd%c%2.2d:%2.2d:%2.2d", + snprintf(time,time_size,"%4.4ld-%2.2hd-%2.2hd%c%2.2d:%2.2d:%2.2d", comptime.year,comptime.month,comptime.day,separator,ihr,imin,isec); break; case 2: - sprintf(time,"%4.4ld-%2.2hd-%2.2hd%c%2.2d:%2.2d", + snprintf(time,time_size,"%4.4ld-%2.2hd-%2.2hd%c%2.2d:%2.2d", comptime.year,comptime.month,comptime.day,separator,ihr,imin); break; case 3: - sprintf(time,"%4.4ld-%2.2hd-%2.2hd%c%2.2d", + snprintf(time,time_size,"%4.4ld-%2.2hd-%2.2hd%c%2.2d", comptime.year,comptime.month,comptime.day,separator,ihr); break; case 4: - sprintf(time,"%4.4ld-%2.2hd-%2.2hd", + snprintf(time,time_size,"%4.4ld-%2.2hd-%2.2hd", comptime.year,comptime.month,comptime.day); break; } @@ -1145,23 +1145,23 @@ cdComp2Iso(cdCalenType timetype, int separator, cdCompTime comptime, char* time) else { /* Climatological */ switch (nskip) { case 0: /* sec != 0 && (int)sec != sec */ - sprintf(time,"%2.2hd-%2.2hd%c%2.2d:%2.2d:%lf", + snprintf(time,time_size,"%2.2hd-%2.2hd%c%2.2d:%2.2d:%lf", comptime.month,comptime.day,separator,ihr,imin,sec); break; case 1: - sprintf(time,"%2.2hd-%2.2hd%c%2.2d:%2.2d:%2.2d", + snprintf(time,time_size,"%2.2hd-%2.2hd%c%2.2d:%2.2d:%2.2d", comptime.month,comptime.day,separator,ihr,imin,isec); break; case 2: - sprintf(time,"%2.2hd-%2.2hd%c%2.2d:%2.2d", + snprintf(time,time_size,"%2.2hd-%2.2hd%c%2.2d:%2.2d", comptime.month,comptime.day,separator,ihr,imin); break; case 3: - sprintf(time,"%2.2hd-%2.2hd%c%2.2d", + snprintf(time,time_size,"%2.2hd-%2.2hd%c%2.2d", comptime.month,comptime.day,separator,ihr); break; case 4: - sprintf(time,"%2.2hd-%2.2hd", + snprintf(time,time_size,"%2.2hd-%2.2hd", comptime.month,comptime.day); break; } @@ -1171,12 +1171,12 @@ cdComp2Iso(cdCalenType timetype, int separator, cdCompTime comptime, char* time) /* rkr: added for output closer to ISO 8601 */ void -cdRel2Iso(cdCalenType timetype, char* relunits, int separator, double reltime, char* chartime) +cdRel2Iso(cdCalenType timetype, char* relunits, int separator, double reltime, char* chartime, size_t chartime_size) { cdCompTime comptime; cdRel2Comp(timetype, relunits, reltime, &comptime); - cdComp2Iso(timetype, separator, comptime, chartime); + cdComp2Iso(timetype, separator, comptime, chartime, chartime_size); return; } diff --git a/ncdump/dumplib.c b/ncdump/dumplib.c index 03fe3b278e..e6ca79a9c9 100644 --- a/ncdump/dumplib.c +++ b/ncdump/dumplib.c @@ -1270,7 +1270,7 @@ nctime_val_tostring(const ncvar_t *varp, safebuf_t *sfbf, const void *valp) { oldopts = cdSetErrOpts(0); newopts = oldopts | CU_VERBOSE; cdSetErrOpts(newopts); - cdRel2Iso(varp->timeinfo->calendar, varp->timeinfo->units, separator, vv, &sout[1]); + cdRel2Iso(varp->timeinfo->calendar, varp->timeinfo->units, separator, vv, &sout[1], sizeof(sout) - 1); cdSetErrOpts(oldopts); res = strlen(sout); sout[res++] = '"'; From 05cd5565e86881dcf0affa1a45f5269505d44ef6 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 12 Dec 2023 09:24:49 -0700 Subject: [PATCH 71/77] Fix typo in nc-config.cmake.in --- nc-config.cmake.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nc-config.cmake.in b/nc-config.cmake.in index 8588665d20..144d9b65bf 100644 --- a/nc-config.cmake.in +++ b/nc-config.cmake.in @@ -118,7 +118,7 @@ fi has_multifilters="yes" -has_zstd="@HAS_ZSTD" +has_zstd="@HAS_ZSTD@" if [ -z "$has_zstd" -o "$has_zstd" = "OFF" -o "$has_zstd" = "FALSE" ]; then has_zstd="no" else From ed90d500784125755a067de01bdc9e5876f69558 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 12 Dec 2023 10:04:18 -0700 Subject: [PATCH 72/77] Typo fix in support of gh2824.wif --- CMakeLists.txt | 5 +++-- nc-config.cmake.in | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 189f6d87c2..51d84cd196 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1160,7 +1160,8 @@ set_std_filter(Szip) SET(HAVE_SZ ${Szip_FOUND}) set_std_filter(Blosc) IF(Zstd_FOUND) -set_std_filter(Zstd) + set_std_filter(Zstd) + SET(HAVE_ZSTD ON CACHE) ENDIF() IF(Bz2_FOUND) set_std_filter(Bz2) @@ -1168,7 +1169,7 @@ ELSE() # The reason we use a local version is to support a more comples test case MESSAGE("libbz2 not found using built-in version") SET(HAVE_LOCAL_BZ2 ON) - SET(HAVE_BZ2 ON) + SET(HAVE_BZ2 ON CACHE BOOL "") set(STD_FILTERS "${STD_FILTERS} bz2") ENDIF() diff --git a/nc-config.cmake.in b/nc-config.cmake.in index 144d9b65bf..15ea967d4c 100644 --- a/nc-config.cmake.in +++ b/nc-config.cmake.in @@ -118,7 +118,7 @@ fi has_multifilters="yes" -has_zstd="@HAS_ZSTD@" +has_zstd="@HAVE_ZSTD@" if [ -z "$has_zstd" -o "$has_zstd" = "OFF" -o "$has_zstd" = "FALSE" ]; then has_zstd="no" else From 61c9a8641d7a4cd55f4ecf95dbd7bf0650a50050 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 12 Dec 2023 10:06:55 -0700 Subject: [PATCH 73/77] Fix cmake syntax typo. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 51d84cd196..c666b42114 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1161,7 +1161,7 @@ SET(HAVE_SZ ${Szip_FOUND}) set_std_filter(Blosc) IF(Zstd_FOUND) set_std_filter(Zstd) - SET(HAVE_ZSTD ON CACHE) + SET(HAVE_ZSTD ON) ENDIF() IF(Bz2_FOUND) set_std_filter(Bz2) From 4673c55a6c53066a55b9de504fc3a2a91e8beded Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 12 Dec 2023 14:03:36 -0700 Subject: [PATCH 74/77] Catching up on PRs, this is 2431 on the current 'main' --- plugins/Makefile.am | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 5bfdad03cc..fc7a984084 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -9,17 +9,7 @@ include $(top_srcdir)/lib_flags.am # for information regarding incrementing `-version-info`. plugin_version_info = -version-info 0:0:0 -AM_LDFLAGS += -module -shared -export-dynamic $(NOUNDEFINED) - -if ISMINGW - AM_LDFLAGS += -avoid-version -else !ISMINGW -if ISCYGWIN -AM_LDFLAGS += -avoid-version -else !ISCYGWIN -AM_LDFLAGS += $(plugin_version_info) -endif !ISCYGWIN -endif !ISMINGW +AM_LDFLAGS += -module -avoid-version -shared -export-dynamic $(NOUNDEFINED) # Create an alternate directory if not installing. ALTPLUGINDIR = ${abs_top_builddir}/plugins/plugindir From 52198b3f123774a9dcfb87b374383872c3a85708 Mon Sep 17 00:00:00 2001 From: wkliao Date: Wed, 20 Apr 2022 00:10:24 -0500 Subject: [PATCH 75/77] count argument in H5Sselect_hyperslab Argument 'count' in NetCDF is not exactly the same as the 'count' in H5Sselect_hyperslabs(space_id, op, start, stride, count, block). When the argument 'stride' is NULL, NetCDF's 'count' should be used in argument 'block', for example, H5Sselect_hyperslabs(space_id, op, start, NULL, ones, count); where 'one' is an array of all 1s. Although using NULL 'block' below H5Sselect_hyperslabs(space_id, op, start, NULL, count, NULL); has the same effect, HDF5 internally stores the space of a subarray as a list of single elements, instead of a "block", which can affect the performance. --- h5_test/tst_h_dimscales2.c | 5 +-- h5_test/tst_h_enums.c | 4 +-- h5_test/tst_h_files.c | 10 +++--- h5_test/tst_h_par.c | 8 +++-- h5_test/tst_h_par_compress.c | 8 +++-- h5_test/tst_h_strings2.c | 7 ++-- h5_test/tst_h_vars.c | 5 +-- libhdf5/hdf5var.c | 37 +++++++++++++++----- libnczarr/zvar.c | 66 +++++++++++++++++++++++++++++++++--- nc_perf/tst_files3.c | 3 +- 10 files changed, 121 insertions(+), 32 deletions(-) diff --git a/h5_test/tst_h_dimscales2.c b/h5_test/tst_h_dimscales2.c index 07592e2063..eb7b2dfb53 100644 --- a/h5_test/tst_h_dimscales2.c +++ b/h5_test/tst_h_dimscales2.c @@ -633,6 +633,7 @@ main() hsize_t h5dimlen[DIMS2], h5dimlenmax[DIMS2], xtend_size[DIMS2] = {1, NUM_VALS}; hsize_t start[DIMS2] = {0, 0}; hsize_t count[DIMS2] = {1, NUM_VALS}; + hsize_t ones[DIMS2] = {1, 1}; double value[NUM_VALS]; int dataset_ndims; int i; @@ -661,7 +662,7 @@ main() /* Set up the file and memory spaces. */ if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if ((mem_spaceid = H5Screate_simple(DIMS2, count, NULL)) < 0) ERR; /* Write a slice of data. */ @@ -683,7 +684,7 @@ main() /* Set up the file and memory spaces for a second slice. */ start[0]++; if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if ((mem_spaceid = H5Screate_simple(DIMS2, count, NULL)) < 0) ERR; /* Write a second slice of data. */ diff --git a/h5_test/tst_h_enums.c b/h5_test/tst_h_enums.c index 352b4dce3b..a107023a4f 100644 --- a/h5_test/tst_h_enums.c +++ b/h5_test/tst_h_enums.c @@ -154,7 +154,7 @@ main() char lang[NUM_LANG][STR_LEN + 1] = {"C", "Fortran", "C++", "MISSING"}; enum langs {CLANG=0, Fortran=1, CPP=2, MISSING=255}; short the_value, fill_value = MISSING, data_point = CLANG; - hsize_t start[1] = {1}, count[1] = {1}; + hsize_t start[1] = {1}, count[1] = {1}, one[1] = {1}; int num_members; size_t size; hid_t base_hdf_typeid; @@ -197,7 +197,7 @@ main() if ((mem_spaceid = H5Screate(H5S_SCALAR)) < 0) ERR; if ((file_spaceid = H5Screate_simple(1, dims, NULL)) < 0) ERR; if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, one, count) < 0) ERR; if (H5Dwrite(datasetid, typeid, mem_spaceid, file_spaceid, H5P_DEFAULT, &data_point) < 0) ERR; diff --git a/h5_test/tst_h_files.c b/h5_test/tst_h_files.c index 15765ea5bd..51d04ca799 100644 --- a/h5_test/tst_h_files.c +++ b/h5_test/tst_h_files.c @@ -174,7 +174,7 @@ main() #define MILLION 1000000 hid_t fileid, write_spaceid, datasetid, mem_spaceid; - hsize_t start[NDIMS], count[NDIMS]; + hsize_t start[NDIMS], count[NDIMS], ones[NDIMS]; hsize_t dims[1]; int *data; int num_steps; @@ -210,8 +210,9 @@ main() { /* Select hyperslab for write of one slice. */ start[0] = s * SC; + ones[0] = 1; if (H5Sselect_hyperslab(write_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if (H5Dwrite(datasetid, H5T_NATIVE_INT, mem_spaceid, write_spaceid, H5P_DEFAULT, data) < 0) ERR; @@ -242,7 +243,7 @@ main() hid_t mem_spaceid, xfer_plistid, native_typeid; hsize_t *chunksize, dims[1], maxdims[1], *dimsize, *maxdimsize; hsize_t fdims[MAX_DIMS], fmaxdims[MAX_DIMS]; - hsize_t start[MAX_DIMS], count[MAX_DIMS]; + hsize_t start[MAX_DIMS], count[MAX_DIMS], ones[MAX_DIMS]; char file_name[STR_LEN + 1]; char dimscale_wo_var[STR_LEN]; void *bufr; @@ -342,7 +343,8 @@ main() start[1] = 0; count[0] = 1; count[1] = 2097153; - if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL, count, NULL) < 0) ERR; + ones[0] = ones[1] = 1; + if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL, ones, count) < 0) ERR; if ((mem_spaceid = H5Screate_simple(NDIMS2, count, NULL)) < 0) ERR; if ((xfer_plistid = H5Pcreate(H5P_DATASET_XFER)) < 0) ERR; if ((native_typeid = H5Tget_native_type(H5T_NATIVE_SCHAR, H5T_DIR_DEFAULT)) < 0) ERR; diff --git a/h5_test/tst_h_par.c b/h5_test/tst_h_par.c index 2d350a4537..e8c9c22912 100644 --- a/h5_test/tst_h_par.c +++ b/h5_test/tst_h_par.c @@ -66,7 +66,7 @@ main(int argc, char **argv) printf("*** Creating file for parallel I/O read, and rereading it..."); { hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; - hsize_t start[NDIMS], count[NDIMS]; + hsize_t start[NDIMS], count[NDIMS], ones[NDIMS]; hsize_t dims[1]; int data[SC1], data_in[SC1]; int num_steps; @@ -126,8 +126,9 @@ main(int argc, char **argv) /* Select hyperslab for write of one slice. */ start[0] = s * SC1 * p + my_rank * SC1; count[0] = SC1; + ones[0] = 1; if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid, xferid, data) < 0) ERR; @@ -185,8 +186,9 @@ main(int argc, char **argv) /* Select hyperslab for read of one slice. */ start[0] = s * SC1 * p + my_rank * SC1; count[0] = SC1; + ones[0] = 1; if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) + start, NULL, ones, count) < 0) { ERR; return 2; diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index 6a15e6a810..e634758032 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -51,7 +51,7 @@ main(int argc, char **argv) { hid_t fapl_id, fileid, whole_spaceid, dsid, slice_spaceid, whole_spaceid1, xferid; hid_t plistid; - hsize_t start[NDIMS], count[NDIMS]; + hsize_t start[NDIMS], count[NDIMS], ones[NDIMS]; hsize_t dims[1], chunksize = SC1; int data[SC1], data_in[SC1]; int num_steps; @@ -120,8 +120,9 @@ main(int argc, char **argv) /* Select hyperslab for write of one slice. */ start[0] = s * SC1 * p + my_rank * SC1; count[0] = SC1; + ones[0] = 1; if (H5Sselect_hyperslab(whole_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; if (H5Dwrite(dsid, H5T_NATIVE_INT, slice_spaceid, whole_spaceid, xferid, data) < 0) ERR; @@ -160,8 +161,9 @@ main(int argc, char **argv) /* Select hyperslab for read of one slice. */ start[0] = s * SC1 * p + my_rank * SC1; count[0] = SC1; + ones[0] = 1; if (H5Sselect_hyperslab(whole_spaceid1, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) + start, NULL, ones, count) < 0) { ERR; return 2; diff --git a/h5_test/tst_h_strings2.c b/h5_test/tst_h_strings2.c index 1d03a983c9..a451a6ac64 100644 --- a/h5_test/tst_h_strings2.c +++ b/h5_test/tst_h_strings2.c @@ -30,6 +30,7 @@ main() hid_t file_spaceid, mem_spaceid; hsize_t dims[1] = {0}, max_dims[1] = {H5S_UNLIMITED}, chunk_dims[1] = {1}; hsize_t xtend_size[NDIMS] = {2}, start[NDIMS] = {1}, count[NDIMS] = {1}; + hsize_t ones[NDIMS] = {1}; /* void *fillp;*/ char *data = "A man who carries a cat by the tail learns " "something he can learn in no other way."; @@ -91,7 +92,7 @@ main() /* Select space in file to write a record. */ if ((file_spaceid = H5Dget_space(datasetid)) < 0) ERR; if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; /* Select space in memory to read from. */ if ((mem_spaceid = H5Screate_simple(NDIMS, count, NULL)) < 0) ERR; @@ -126,7 +127,7 @@ main() hid_t typeid, datasetid, plistid; hid_t file_spaceid, mem_spaceid; hsize_t dims[1] = {2}, chunk_dims[1] = {1}; - hsize_t start[NDIMS] = {1}, count[NDIMS] = {1}; + hsize_t start[NDIMS] = {1}, count[NDIMS] = {1}, ones[NDIMS] = {1}; /* void *fillp;*/ char *data = "A man who carries a cat by the tail learns " "something he can learn in no other way."; @@ -179,7 +180,7 @@ To be good is noble; but to show others how to be good is nobler and no trouble. /* Select space in file to write a record. */ if ((file_spaceid = H5Dget_space(datasetid)) < 0) ERR; if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; /* Select space in memory to read from. */ if ((mem_spaceid = H5Screate_simple(NDIMS, count, NULL)) < 0) diff --git a/h5_test/tst_h_vars.c b/h5_test/tst_h_vars.c index 7fa840f412..a74ab50472 100644 --- a/h5_test/tst_h_vars.c +++ b/h5_test/tst_h_vars.c @@ -69,7 +69,7 @@ main() float float_data_out[LAT_LEN][LON_LEN]; hsize_t dims[NDIMS], max_dims[NDIMS]; hsize_t dims_in[NDIMS], max_dims_in[NDIMS]; - hsize_t start[MAX_DIMS], count[MAX_DIMS]; + hsize_t start[MAX_DIMS], count[MAX_DIMS], ones[MAX_DIMS]; int lat, lon; /* Set up some phoney data, 1 record's worth. In C, first @@ -153,8 +153,9 @@ main() start[0] = 1; start[1] = 0; start[2] = 0; + ones[0] = ones[1] = ones[2] = 1; if (H5Sselect_hyperslab(write_spaceid, H5S_SELECT_SET, - start, NULL, count, NULL) < 0) ERR; + start, NULL, ones, count) < 0) ERR; /* Write second record of data to each dataset. */ if (H5Dwrite(pres_dsid, H5T_IEEE_F32LE, mem_spaceid, write_spaceid, diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 6c2303ec45..d08cfca8a0 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1542,11 +1542,12 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, NC_VAR_INFO_T *var; NC_DIM_INFO_T *dim; NC_HDF5_VAR_INFO_T *hdf5_var; + herr_t herr; hid_t file_spaceid = 0, mem_spaceid = 0, xfer_plistid = 0; long long unsigned xtend_size[NC_MAX_VAR_DIMS]; hsize_t fdims[NC_MAX_VAR_DIMS], fmaxdims[NC_MAX_VAR_DIMS]; hsize_t start[NC_MAX_VAR_DIMS], count[NC_MAX_VAR_DIMS]; - hsize_t stride[NC_MAX_VAR_DIMS]; + hsize_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS]; int need_to_extend = 0; #ifdef USE_PARALLEL4 int extend_possible = 0; @@ -1597,6 +1598,7 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp ? countp[i] : var->dim[i]->len; stride[i] = stridep ? stridep[i] : 1; + ones[i] = 1; LOG((4, "start[%d] %ld count[%d] %ld stride[%d] %ld", i, start[i], i, count[i], i, stride[i])); /* Check to see if any counts are zero. */ @@ -1647,8 +1649,13 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, } else { - if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, stride, - count, NULL) < 0) + if (stridep == NULL) + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, + NULL, ones, count); + else + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, + stride, count, NULL); + if (herr < 0) BAIL(NC_EHDFERR); /* Create a space for the memory, just big enough to hold the slab @@ -1773,8 +1780,14 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, BAIL2(NC_EHDFERR); if ((file_spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0) BAIL(NC_EHDFERR); - if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, stride, count, NULL) < 0) + + if (stridep == NULL) + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, NULL, ones, count); + else + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, stride, count, NULL); + if (herr < 0) BAIL(NC_EHDFERR); } } @@ -1873,7 +1886,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, hsize_t count[NC_MAX_VAR_DIMS]; hsize_t fdims[NC_MAX_VAR_DIMS], fmaxdims[NC_MAX_VAR_DIMS]; hsize_t start[NC_MAX_VAR_DIMS]; - hsize_t stride[NC_MAX_VAR_DIMS]; + hsize_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS]; void *fillvalue = NULL; int no_read = 0, provide_fill = 0; hssize_t fill_value_size[NC_MAX_VAR_DIMS]; @@ -1924,6 +1937,7 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp[i]; stride[i] = stridep ? stridep[i] : 1; + ones[i] = 1; /* if any of the count values are zero don't actually read. */ if (count[i] == 0) @@ -2053,9 +2067,16 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, } else { - if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, stride, count, NULL) < 0) + herr_t herr; + if (stridep == NULL) + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, NULL, ones, count); + else + herr = H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, stride, count, NULL); + if (herr < 0) BAIL(NC_EHDFERR); + /* Create a space for the memory, just big enough to hold the slab we want. */ if ((mem_spaceid = H5Screate_simple(var->ndims, count, NULL)) < 0) diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 5076ff740a..df06d9dfb0 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -1546,7 +1546,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, #endif size64_t fdims[NC_MAX_VAR_DIMS]; size64_t start[NC_MAX_VAR_DIMS], count[NC_MAX_VAR_DIMS]; - size64_t stride[NC_MAX_VAR_DIMS]; + size64_t stride[NC_MAX_VAR_DIMS], ones[NC_MAX_VAR_DIMS]; int retval, range_error = 0, i, d2; void *bufr = NULL; int bufrd = 0; /* 1 => we allocated bufr */ @@ -1591,6 +1591,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[0] = 0; count[0] = 1; stride[0] = 1; + ones[0] = 1; } else { for (i = 0; i < var->ndims; i++) { @@ -1602,6 +1603,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp ? countp[i] : fdims[i]; stride[i] = stridep ? stridep[i] : 1; + ones[i] = 1; /* Check to see if any counts are zero. */ if (!count[i]) @@ -1633,6 +1635,31 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, } } + +#ifdef LOOK + /* Now you would think that no one would be crazy enough to write + a scalar dataspace with one of the array function calls, but you + would be wrong. So let's check to see if the dataset is + scalar. If it is, we won't try to set up a hyperslab. */ + if (H5Sget_simple_extent_type(file_spaceid) == H5S_SCALAR) + { + if ((mem_spaceid = H5Screate(H5S_SCALAR)) < 0) + BAIL(NC_EHDFERR); + } + else + { + if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, stride, + ones, count) < 0) + BAIL(NC_EHDFERR); + + /* Create a space for the memory, just big enough to hold the slab + we want. */ + if ((mem_spaceid = H5Screate_simple(var->ndims, count, NULL)) < 0) + BAIL(NC_EHDFERR); + } +#endif + + /* Are we going to convert any data? (No converting of compound or * opaque or vlen types.) We also need to call this code if we are doing * quantization. */ @@ -1699,6 +1726,30 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, } } } + +#endif + +#ifdef LOOK + /* If we need to extend it, we also need a new file_spaceid + to reflect the new size of the space. */ + if (need_to_extend) + { + LOG((4, "extending dataset")); + /* Convert xtend_size back to hsize_t for use with + * H5Dset_extent. */ + for (d2 = 0; d2 < var->ndims; d2++) + fdims[d2] = (size64_t)xtend_size[d2]; + if (H5Dset_extent(ncz_var->hdf_datasetid, fdims) < 0) + BAIL(NC_EHDFERR); + if (file_spaceid > 0 && H5Sclose(file_spaceid) < 0) + BAIL2(NC_EHDFERR); + if ((file_spaceid = H5Dget_space(ncz_var->hdf_datasetid)) < 0) + BAIL(NC_EHDFERR); + if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, + start, stride, ones, count) < 0) + BAIL(NC_EHDFERR); + } +#endif } /* Do we need to convert the data? */ @@ -1801,8 +1852,8 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, size64_t fdims[NC_MAX_VAR_DIMS]; /* size of the dimensions */ size64_t start[NC_MAX_VAR_DIMS]; size64_t stride[NC_MAX_VAR_DIMS]; - int no_read = 0; - int provide_fill = 0; + size64_t ones[NC_MAX_VAR_DIMS]; + int no_read = 0, provide_fill = 0; int fill_value_size[NC_MAX_VAR_DIMS]; int retval, range_error = 0, i, d2; void *bufr = NULL; @@ -1834,6 +1885,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[0] = 0; count[0] = 1; stride[0] = 1; + ones[0] = 1; } else { for (i = 0; i < var->ndims; i++) { @@ -1843,6 +1895,12 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, start[i] = startp[i]; count[i] = countp[i]; stride[i] = stridep ? stridep[i] : 1; + + ones[i] = 1; + /* if any of the count values are zero don't actually read. */ + if (count[i] == 0) + no_read++; + /* Get dimension sizes also */ fdims[i] = var->dim[i]->len; /* if any of the counts are zero don't actually read. */ @@ -1960,7 +2018,7 @@ NCZ_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, else { if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, - start, stride, count, NULL) < 0) + start, stride, ones, count) < 0) BAIL(NC_EHDFERR); /* Create a space for the memory, just big enough to hold the slab we want. */ diff --git a/nc_perf/tst_files3.c b/nc_perf/tst_files3.c index ab04f7ec06..f4404e0535 100644 --- a/nc_perf/tst_files3.c +++ b/nc_perf/tst_files3.c @@ -100,6 +100,7 @@ int dump_hdf_file(const float *data, int docompression) hsize_t dims[NDIMS] = {X_LEN, Y_LEN, Z_LEN}; hsize_t start[NDIMS] = {0, 0, 0}; hsize_t count[NDIMS] = {1, 1, Z_LEN}; + hsize_t ones[NDIMS] = {1, 1, 1}; /* create file */ file_id = H5Fcreate(FILE_NAME, H5F_ACC_TRUNC, @@ -138,7 +139,7 @@ int dump_hdf_file(const float *data, int docompression) for (start[1] = 0; start[1] < Y_LEN; start[1]++) { if (H5Sselect_hyperslab(file_spaceid, H5S_SELECT_SET, start, NULL, - count, NULL) < 0) ERR_RET; + ones, count) < 0) ERR_RET; if (H5Dwrite(dataset_id, H5T_NATIVE_FLOAT, mem_spaceid, file_spaceid, xfer_plistid, data) < 0) ERR_RET; } From 6a628b9ca7b8c027d6d6dd0327ab41597f0b0447 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Tue, 12 Dec 2023 16:49:13 -0700 Subject: [PATCH 76/77] Rebased PR by hand against main. --- libnczarr/zvar.c | 3 ++- nc_test4/tst_virtual_datasets.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index df06d9dfb0..607a351858 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -1703,6 +1703,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, do no harm to reextend it to that size. */ if (var->ndims) { + for (d2 = 0; d2 < var->ndims; d2++) { size64_t endindex = start[d2] + stride[d2] * (count[d2] - 1); /* last index written */ @@ -1727,7 +1728,7 @@ NCZ_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, } } -#endif + #ifdef LOOK /* If we need to extend it, we also need a new file_spaceid diff --git a/nc_test4/tst_virtual_datasets.c b/nc_test4/tst_virtual_datasets.c index acc3c1ce3c..ff278a3dde 100644 --- a/nc_test4/tst_virtual_datasets.c +++ b/nc_test4/tst_virtual_datasets.c @@ -54,7 +54,7 @@ int read_back_contents(const char* filename) { int ncid, varid, ndims; size_t i; float data[VARIABLE_SIZE]; - size_t i; + char var_name[NC_MAX_NAME+1]; int dimids_var[1], var_type, natts; From e662790a6658e8c664d1bb94f7e83201cd500818 Mon Sep 17 00:00:00 2001 From: Ward Fisher Date: Thu, 14 Dec 2023 10:46:19 -0700 Subject: [PATCH 77/77] Changed link to netCDF-Fortran documentation. --- docs/mainpage.dox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mainpage.dox b/docs/mainpage.dox index 64c2df84ac..0284bd8318 100644 --- a/docs/mainpage.dox +++ b/docs/mainpage.dox @@ -13,7 +13,7 @@ The NetCDF homepage may be found at The NetCDF-Fortran Developer's Guide +- The NetCDF-Fortran Developer's Guide \section this_release Learn more about the current NetCDF-C Release