[cache] Validate cached file with hashlib instead of dask tokenize

[bdestombe]

No description provided.

[dbrakenhoff]

Does this make #393 unnecessary?

[bdestombe]

Yes.

It works in my script and I tested it with the code of Onno.
hashlib_hash2 = hashlib.sha256(open(fp_cache, 'rb').read()).hexdigest() the hash returned by the first script is the same as the one returned by the second script

[bdestombe]

The adjusted code of Onno

File 1

import os

import nlmod
import pickle
import xarray as xr
import dask
import hashlib

# instellingen voor de logging
nlmod.util.get_color_logger("INFO")

cachedir ='.'
extent = [204800, 205000, 438000, 438200]

#%%
# get regis dataset
fp_cache = "regis.nc"
if os.path.exists(fp_cache):
   os.remove(fp_cache)

regis_ds = nlmod.read.regis.get_regis(extent, cachedir=cachedir, cachename="regis")
hash_orig = dask.base.tokenize(regis_ds.to_dict())
hashlib_hash1 = hashlib.sha256(open(fp_cache, 'rb').read()).hexdigest()

# read cached netcdf
regis_from_cache = xr.open_dataset('regis.nc')
hash_cache_direct = dask.base.tokenize(regis_from_cache.to_dict())

# read pickle
with open('regis_ds.pickle', 'rb') as handle:
    regis_from_pickle = pickle.load(handle)
hash_pickle = dask.base.tokenize(regis_from_pickle.to_dict())

# call get_regis again (this time the cache will be used)
regis_ds_ind = nlmod.read.regis.get_regis(extent, cachedir=cachedir, cachename="regis")
hash_cache_indirect = dask.base.tokenize(regis_ds_ind.to_dict())

# save hashes
with open('hashes.txt', 'w') as handle:
    handle.write(f'regis orig           : {hash_orig}\n')
    handle.write(f'regis cache direct   : {hash_cache_direct}\n')
    handle.write(f'regis pickle         : {hash_pickle}\n')
    handle.write(f'regis cache indirect : {hash_cache_indirect}\n')
    handle.write(f"hashlib1             : {hashlib_hash1}")

File 2

import nlmod
import pickle
import xarray as xr
import dask
import hashlib

# instellingen voor de logging
nlmod.util.get_color_logger("INFO")

cachedir ='.'
extent = [204800, 205000, 438000, 438200]

# read cached netcdf
fp_cache = "regis.nc"
hashlib_hash2 = hashlib.sha256(open(fp_cache, 'rb').read()).hexdigest()
regis_from_cache = xr.open_dataset('regis.nc')
hash_cache_direct = dask.base.tokenize(regis_from_cache.to_dict())
regis_from_cache.close()

# read pickle
with open('regis_ds.pickle', 'rb') as handle:
    regis_from_pickle = pickle.load(handle)
hash_pickle = dask.base.tokenize(regis_from_pickle.to_dict())

# call get_regis again (this time the cache won't be used)
regis_ds_ind = nlmod.read.regis.get_regis(extent, cachedir=cachedir, cachename="regis")
hash_cache_indirect = dask.base.tokenize(regis_ds_ind.to_dict())

# save hashes
with open('hashesv2.txt', 'w') as handle:
    handle.write(f'regis cache direct   : {hash_cache_direct}\n')
    handle.write(f'regis pickle         : {hash_pickle}\n')
    handle.write(f'regis failed cache   : {hash_cache_indirect}\n')
    handle.write(f"hashlib2             : {hashlib_hash2}")

hashlib_hash1 is the same as hashlib_hash2