Skip to content

Commit

Permalink
Fixes to netCDF4 and hdf5 (#1926)
Browse files Browse the repository at this point in the history 
Co-authored-by: Richard Siddans <[email protected]>
Co-authored-by: Giloo <[email protected]>
  • Loading branch information
@rsiddans @GillesDuvert
3 people authored Nov 28, 2024
1 parent db3616e commit 430c1ee
Show file tree
Hide file tree
Showing 6 changed files with 271 additions and 26 deletions.
30 changes: 21 additions & 9 deletions src/hdf5_fun.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,23 +155,23 @@ namespace lib {
// { H5T_NATIVE_LDOUBLE }, //not until LDOUBLE Is handled everywhere!!

/* GDL_DOUBLE */
{ H5T_NATIVE_DOUBLE },
{ H5T_NATIVE_DOUBLE,H5T_IEEE_F64BE,H5T_IEEE_F64LE,H5T_INTEL_F64,H5T_MIPS_F64,H5T_ALPHA_F64},

/* GDL_FLOAT */
{ H5T_NATIVE_FLOAT },
{ H5T_NATIVE_FLOAT,H5T_ALPHA_F32,H5T_IEEE_F32BE, H5T_IEEE_F32LE,H5T_INTEL_F32,H5T_MIPS_F32},

/* GDL_ULONG64 */
{ H5T_NATIVE_ULLONG, H5T_ALPHA_U64, H5T_INTEL_U64, H5T_MIPS_U64,
H5T_NATIVE_UINT64, H5T_NATIVE_UINT_FAST64, H5T_NATIVE_UINT_LEAST64,
H5T_STD_U64BE, H5T_STD_U64LE },

/* GDL_LONG64 */
{ H5T_NATIVE_LLONG, H5T_IEEE_F64BE, H5T_IEEE_F64LE, H5T_INTEL_B64,
H5T_INTEL_F64, H5T_INTEL_I64, H5T_MIPS_B64, H5T_MIPS_F64,
{ H5T_NATIVE_LLONG, H5T_INTEL_B64,
H5T_INTEL_I64, H5T_MIPS_B64,
H5T_MIPS_I64, H5T_NATIVE_B64, H5T_NATIVE_INT64, H5T_NATIVE_INT_FAST64,
H5T_NATIVE_INT_LEAST64, H5T_STD_B64BE, H5T_STD_B64LE, H5T_STD_I64BE,
H5T_STD_I64LE, H5T_UNIX_D64BE, H5T_UNIX_D64LE, H5T_ALPHA_B64,
H5T_ALPHA_F64, H5T_ALPHA_I64 },
H5T_ALPHA_I64 },

/* GDL_ULONG */
{ H5T_NATIVE_ULONG, H5T_ALPHA_U32, H5T_INTEL_U32, H5T_MIPS_U32,
Expand All @@ -181,9 +181,9 @@ namespace lib {
/* GDL_LONG */
{ /// H5T_NATIVE_HBOOL,
/// ^--- disabled as it matches against 'H5T_STD_U8LE' (GDL_BYTE)
H5T_NATIVE_LONG, H5T_ALPHA_B32, H5T_ALPHA_F32,
H5T_ALPHA_I32, H5T_IEEE_F32BE, H5T_IEEE_F32LE, H5T_INTEL_B32,
H5T_INTEL_F32, H5T_INTEL_I32, H5T_MIPS_B32, H5T_MIPS_F32,
H5T_NATIVE_LONG, H5T_ALPHA_B32,
H5T_ALPHA_I32, H5T_INTEL_B32,
H5T_INTEL_I32, H5T_MIPS_B32,
H5T_MIPS_I32, H5T_NATIVE_B32, H5T_NATIVE_INT32, H5T_NATIVE_INT_FAST32,
H5T_NATIVE_INT_LEAST32, H5T_STD_B32BE, H5T_STD_B32LE, H5T_STD_I32BE,
H5T_STD_I32LE, H5T_UNIX_D32BE, H5T_UNIX_D32LE },
Expand Down Expand Up @@ -863,7 +863,8 @@ namespace lib {

} else if (ourType == GDL_STRING) {

if (debug) printf("fixed-length string dataset\n");
bool isVarLenStr = H5Tis_variable_str(elem_dtype) > 0;
if (debug) printf(isVarLenStr ? "variable-length string dataset\n" : "fixed-length string dataset\n");

// string length (terminator included)
SizeT str_len = H5Tget_size(elem_dtype);
Expand All @@ -872,6 +873,17 @@ namespace lib {
SizeT num_elems=1;
for(int i=0; i<rank_s; i++) num_elems *= count_s[i];

if (num_elems == 1 && isVarLenStr) {
char* raw = nullptr;
hdf5_basic_read( loc_id, datatype, ms_id, fs_id, &raw, e );

// create GDL variable
res = new DStringGDL(raw);

H5Dvlen_reclaim (ms_id, fs_id, H5P_DEFAULT, &raw);

return res;
}
// allocate & read raw buffer
char* raw = (char*) malloc(num_elems*str_len*sizeof(char));
hdf5_name_guard raw_guard = hdf5_name_guard(raw);
Expand Down
4 changes: 2 additions & 2 deletions src/libinit_cl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ void LibInit_cl()

//writing NetCDF
//Put attribute into file
const string ncdf_attputKey[]={"GLOBAL","LENGTH","BYTE","CHAR","DOUBLE","FLOAT","LONG","SHORT",KLISTEND};
const string ncdf_attputKey[]={"GLOBAL","LENGTH","BYTE","CHAR","DOUBLE","FLOAT","LONG","SHORT","UBYTE","ULONG","USHORT","INT64","UINT64","STRING",KLISTEND};
new DLibPro(lib::ncdf_attput, string("NCDF_ATTPUT"), 4,ncdf_attputKey);

//copy attribute between files
Expand All @@ -174,7 +174,7 @@ void LibInit_cl()
new DLibPro(lib::ncdf_dimrename, string("NCDF_DIMRENAME"), 3);

//define variable
const string ncdf_vardefKey[]={"BYTE","CHAR","DOUBLE","FLOAT","LONG","SHORT",
const string ncdf_vardefKey[]={"BYTE","CHAR","DOUBLE","FLOAT","LONG","SHORT","UBYTE","ULONG","USHORT","INT64","UINT64","STRING","GZIP","SHUFFLE",
KLISTEND};
new DLibFunRetNew(lib::ncdf_vardef, string("NCDF_VARDEF"), 3,ncdf_vardefKey);

Expand Down
126 changes: 120 additions & 6 deletions src/ncdf_att_cl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,12 +219,45 @@ namespace lib {
delete e->GetParGlobal(nParam-1);
e->GetParGlobal(nParam-1)=temp;
}
else if (att_type == NC_STRING) {
// Read NC_STRING
char **stemp = new char*[length];
// Read the (possible array of) strings - nc_get_att_string allocates memory for the strings themselves
status = nc_get_att_string(cdfid, varid, attname.c_str(), stemp);
ncdf_handle_error(e, status, "NCDF_ATTGET");
delete e->GetParGlobal(nParam-1);
// This will only copy first string of array to the returned variable...
e->GetParGlobal(nParam-1)=new DStringGDL(stemp[0]);
// Free the string data from the stemp (having copied the string to the output)
status = nc_free_string(length, stemp);
ncdf_handle_error(e, status, "NCDF_ATTGET");
}
else
{
dimension dim(length);
BaseGDL* temp;
switch (att_type)
{
case NC_INT64 :
{
long long int *i64p = new long long int[length];
status=nc_get_att_longlong(cdfid, varid, attname.c_str(), i64p);
ncdf_att_handle_error(e, status, "NCDF_ATTGET", i64p);
temp = length == 1 ? new DLong64GDL(BaseGDL::NOZERO) : new DLong64GDL(dim, BaseGDL::NOZERO);
memcpy(&(*static_cast<DLong64GDL*>(temp))[0], &(*i64p), length * sizeof(long long int));
delete[] i64p;
break;
}
case NC_UINT64 :
{
unsigned long long int *ui64p = new unsigned long long int[length];
status=nc_get_att_ulonglong(cdfid, varid, attname.c_str(), ui64p);
ncdf_att_handle_error(e, status, "NCDF_ATTGET", ui64p);
temp = length == 1 ? new DULong64GDL(BaseGDL::NOZERO) : new DULong64GDL(dim, BaseGDL::NOZERO);
memcpy(&(*static_cast<DULong64GDL*>(temp))[0], &(*ui64p), length * sizeof(unsigned long long int));
delete[] ui64p;
break;
}
case NC_INT :
{
int *ip = new int[length];
Expand All @@ -235,6 +268,16 @@ namespace lib {
delete[] ip;
break;
}
case NC_UINT :
{
unsigned int *uip = new unsigned int[length];
status=nc_get_att_uint(cdfid, varid, attname.c_str(), uip);
ncdf_att_handle_error(e, status, "NCDF_ATTGET", uip);
temp = length == 1 ? new DULongGDL(BaseGDL::NOZERO) : new DULongGDL(dim, BaseGDL::NOZERO);
memcpy(&(*static_cast<DULongGDL*>(temp))[0], &(*uip), length * sizeof(unsigned int));
delete[] uip;
break;
}
case NC_SHORT :
{
short *sp = new short[length];
Expand All @@ -245,6 +288,16 @@ namespace lib {
delete[] sp;
break;
}
case NC_USHORT :
{
unsigned short *usp = new unsigned short[length];
status = nc_get_att_ushort(cdfid, varid, attname.c_str(), usp);
ncdf_att_handle_error(e, status, "NCDF_ATTGET", usp);
temp = length == 1 ? new DIntGDL(BaseGDL::NOZERO) : new DIntGDL(dim, BaseGDL::NOZERO);
memcpy(&(*static_cast<DUIntGDL*>(temp))[0], &(*usp), length * sizeof(DUInt));
delete[] usp;
break;
}
case NC_FLOAT :
{
float *fp = new float[length];
Expand All @@ -266,9 +319,15 @@ namespace lib {
break;
}
case NC_BYTE :
case NC_UBYTE :
{
unsigned char *bp = new unsigned char[length];
status = nc_get_att_uchar(cdfid, varid, attname.c_str(), bp);
if (status == NC_ERANGE) {
// just warn on NC_ERANGE (probably intended to if idl char used to represent signed byte)
//Warning("Warning in NCDF_ATTGET: NC_ERANGE while reading BYTE");
status=NC_NOERR;
}
ncdf_att_handle_error(e, status, "NCDF_ATTGET", bp);
temp = length == 1 ? new DByteGDL(BaseGDL::NOZERO) : new DByteGDL(dim, BaseGDL::NOZERO);
memcpy(&(*static_cast<DByteGDL*>(temp))[0], &(*bp), length * sizeof(DByte));
Expand Down Expand Up @@ -330,10 +389,16 @@ namespace lib {
if (val->Type() == GDL_BYTE) xtype=NC_BYTE;
if (val->Type() == GDL_STRING) xtype=NC_CHAR;
if (val->Type() == GDL_INT) xtype=NC_SHORT;
if (val->Type() == GDL_UINT) xtype=NC_USHORT;
if (val->Type() == GDL_LONG) xtype=NC_INT;
if (val->Type() == GDL_ULONG) xtype=NC_UINT;
if (val->Type() == GDL_LONG64) xtype=NC_INT64;
if (val->Type() == GDL_ULONG64) xtype=NC_UINT64;
if (val->Type() == GDL_FLOAT) xtype=NC_FLOAT;
if (val->Type() == GDL_DOUBLE) xtype=NC_DOUBLE;
// SA: TODO: GDL_UINT, GDL_ULONG, GDL_COMPLEX, GDL_PTR...
//
// "BYTE","CHAR","DOUBLE","FLOAT","LONG","SHORT","UBYTE","ULONG","USHORT","INT64","UINT64","STRING"

if(e->KeywordSet(2)) //GDL_BYTE
xtype=NC_BYTE;
Expand All @@ -347,6 +412,18 @@ namespace lib {
xtype=NC_INT;
else if(e->KeywordSet(7)) //SHORT
xtype=NC_SHORT;
else if(e->KeywordSet(8))
xtype=NC_UBYTE;
else if(e->KeywordSet(9))
xtype=NC_UINT;
else if(e->KeywordSet(10))
xtype=NC_USHORT;
else if(e->KeywordSet(11))
xtype=NC_INT64;
else if(e->KeywordSet(12))
xtype=NC_UINT64;
else if(e->KeywordSet(13))
xtype=NC_STRING;

// LENGTH keyword support
DLong length;
Expand All @@ -367,6 +444,11 @@ namespace lib {
attname.c_str(),xtype,
(size_t)length,
(const unsigned char *)&(*bvar)[0]);
if (status == NC_ERANGE) {
// just warn on NC_ERANGE (probably intended to if idl char used to represent signed byte)
//Warning("Warning in NCDF_ATTPUT: NC_ERANGE while writing BYTE");
status=NC_NOERR;
}
}
else if(val->Type() == GDL_STRING)
{
Expand All @@ -375,12 +457,17 @@ namespace lib {

length = cvar.length();
e->AssureLongScalarKWIfPresent(1, length);
if (length > cvar.length()) e->Throw("LENGTH keyword value (" + i2s(length) +
") exceedes the data length (" + i2s(cvar.length()) + ")");
if (length < cvar.length()) cvar.resize(length);

status=nc_put_att_text(cdfid,varid, attname.c_str(),
if ( xtype == NC_STRING ) {
char *tmp = (char *)cvar.c_str();
status=nc_put_att_string(cdfid,varid, attname.c_str(),
1, (const char**)&tmp);
} else {
if (length > cvar.length()) e->Throw("LENGTH keyword value (" + i2s(length) +
") exceedes the data length (" + i2s(cvar.length()) + ")");
if (length < cvar.length()) cvar.resize(length);
status=nc_put_att_text(cdfid,varid, attname.c_str(),
cvar.length(), (char *)cvar.c_str());
}
}
else if(val->Type() == GDL_INT)
{
Expand All @@ -389,13 +476,41 @@ namespace lib {
attname.c_str(), xtype,
(size_t)length, &(*ivar)[0]);
}
else if(val->Type() == GDL_UINT)
{
DUIntGDL * uivar=static_cast<DUIntGDL*>(val);
status=nc_put_att_ushort(cdfid,varid,
attname.c_str(), xtype,
(size_t)length, &(*uivar)[0]);
}
else if(val->Type() == GDL_LONG)
{
DLongGDL * lvar=static_cast<DLongGDL*>(val);
status=nc_put_att_int(cdfid,varid,
attname.c_str(),xtype,
(size_t)length, &(*lvar)[0]);
}
else if(val->Type() == GDL_ULONG)
{
DULongGDL * ulvar=static_cast<DULongGDL*>(val);
status=nc_put_att_uint(cdfid,varid,
attname.c_str(),xtype,
(size_t)length, &(*ulvar)[0]);
}
else if(val->Type() == GDL_LONG64)
{
DLong64GDL * l64var=static_cast<DLong64GDL*>(val);
status=nc_put_att_longlong(cdfid,varid,
attname.c_str(),xtype,
(size_t)length, &(*l64var)[0]);
}
else if(val->Type() == GDL_ULONG64)
{
DULong64GDL * ul64var=static_cast<DULong64GDL*>(val);
status=nc_put_att_ulonglong(cdfid,varid,
attname.c_str(),xtype,
(size_t)length, &(*ul64var)[0]);
}
else if(val->Type() == GDL_FLOAT)
{
DFloatGDL * fvar=static_cast<DFloatGDL*>(val);
Expand All @@ -410,7 +525,6 @@ namespace lib {
attname.c_str(),xtype,
(size_t)length, &(*dvar)[0]);
}

ncdf_handle_error(e, status,"NCDF_ATTPUT");

return;
Expand Down
4 changes: 4 additions & 0 deletions src/ncdf_cl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,15 @@ namespace lib {
switch (vartype)
{
case NC_BYTE: return DStringGDL("BYTE");//8 bit
case NC_UBYTE: return DStringGDL("UBYTE");//8 bit
case NC_CHAR: return DStringGDL("CHAR");//8 bit as string
case NC_SHORT: return DStringGDL("INT");//16 bit
case NC_USHORT: return DStringGDL("UINT");//16 bit
case NC_INT: return DStringGDL("LONG");//32 bit
case NC_UINT: return DStringGDL("ULONG");//32 bit
case NC_FLOAT: return DStringGDL("FLOAT");//32 bit float
case NC_DOUBLE:return DStringGDL("DOUBLE");//64 bit double
case NC_STRING:return DStringGDL("STRING");//String
}
return DStringGDL("UNKNOWN");
}
Expand Down
Loading

0 comments on commit 430c1ee

Please sign in to comment.