Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#350: override file_location #351

Merged
merged 3 commits into from
Jan 6, 2025
Merged

#350: override file_location #351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c23b1f3
#350: override file_location
mortenwh Dec 19, 2024
cd03c7c
#350: fix flake error
mortenwh Dec 19, 2024
8a44ba3
#350: add script argument and test
mortenwh Jan 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion py_mmd_tools/nc_to_mmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1657,6 +1657,10 @@ def to_mmd(
platform : dict
A dictionary specifying the platform according to MMD
guidelines.
file_location : str
An alternative to the current file location. This can
be used if the NetCDF file will be moved after the
MMD file has been created.

This list can be extended but requires some new code...
"""
Expand All @@ -1671,6 +1675,7 @@ def to_mmd(
geographic_extent_rectangle = overrides.pop("geographic_extent_rectangle", None)
dataset_citation = overrides.pop("dataset_citation", None)
platform = overrides.pop("platform", None)
file_location = overrides.pop("file_location", os.path.dirname(self.netcdf_file))

# Get ncin object from instance
ncin = self.ncin
Expand Down Expand Up @@ -1836,7 +1841,7 @@ def to_mmd(
# Set storage_information
self.metadata["storage_information"] = {
"file_name": os.path.basename(self.netcdf_file),
"file_location": os.path.dirname(self.netcdf_file),
"file_location": file_location,
"file_format": "NetCDF-CF",
"file_size": "%.2f" % self.file_size,
"file_size_unit": "MB",
Expand Down
86 changes: 47 additions & 39 deletions py_mmd_tools/script/nc2mmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,64 +32,68 @@
def create_parser():
"""Create parser object"""
parser = argparse.ArgumentParser(
description='Create an MMD xml file from an input netCDF file.'
description="Create an MMD xml file from an input netCDF file."
)

# Add to parse a whole server?
parser.add_argument(
'-i', '--input', type=str,
help='Input file or folder.'
"-i", "--input", type=str,
help="Input file or folder."
)
parser.add_argument(
'--dry-run', action='store_true',
help='Dry-run to check the netcdf metadata content.'
"--dry-run", action="store_true",
help="Dry-run to check the netcdf metadata content."
)
parser.add_argument(
'-u', '--url', type=str,
help='OPeNDAP url. If multiple files shall be processed, '
'this should be their base url.'
"-u", "--url", type=str,
help="OPeNDAP url. If multiple files shall be processed, "
"this should be their base url."
)
parser.add_argument(
'-o', '--output_dir', type=pathlib.Path,
help='Output directory.'
"-o", "--output_dir", type=pathlib.Path,
help="Output directory."
)
parser.add_argument(
'-w', '--add_wms_data_access', action='store_true',
help='Add wms data access (optional).'
"-w", "--add_wms_data_access", action="store_true",
help="Add wms data access (optional)."
)
parser.add_argument(
'-l', '--wms_link', default=None,
help=('Specify a custom WMS link. '
'Default will generate a link to ncwms based on the input data. '
'Should be use together with wms_layer_names. '
'Please note \'?service=WMS&version=1.3.0&request=GetCapabilities\' '
'will be added to the link automatically.')
"-l", "--wms_link", default=None,
help=("Specify a custom WMS link. "
"Default will generate a link to ncwms based on the input data. "
"Should be use together with wms_layer_names. "
"Please note \'?service=WMS&version=1.3.0&request=GetCapabilities\' "
"will be added to the link automatically.")
)
parser.add_argument(
'-n', '--wms_layer_names', default=[], nargs='*',
help=('Specify a custom WMS layer names. Default will use the netcdf variable names as '
'layer names. Only applied if wms_link also is given')
"-n", "--wms_layer_names", default=[], nargs="*",
help=("Specify custom WMS layer names. Default will use the netcdf variable names as "
"layer names. Only applied if wms_link also is given")
)
parser.add_argument(
'-c', '--checksum_calculation', action='store_true',
"-c", "--checksum_calculation", action="store_true",
help="Toggle whether to calculate the checksum of the file"
)
parser.add_argument(
'--collection', default=None,
"--collection", default=None,
help="Specify MMD collection field (default is METNCS)"
)
parser.add_argument(
'--parent', default=None,
"--parent", default=None,
help="Metadata ID of a parent dataset"
)
parser.add_argument(
'--log-ids', default=None,
help='Store the metadata IDs in a file'
"--log-ids", default=None,
help="Store the metadata IDs in a file"
)
parser.add_argument(
'--print_warnings', action='store_true',
"--print_warnings", action="store_true",
help="Toggle whether to print warnings"
)
parser.add_argument(
"--file_location", default=None,
help=("Optionally, provide the CF-NetCDF file location (e.g., if the file will be moved "
"after creation). By default, the existing file location be used."))

return parser

Expand All @@ -98,25 +102,25 @@ def main(args=None):
"""Run tool to create MMD xml file from input netCDF-CF file(s)"""
if not args.dry_run:
if args.url is None:
raise ValueError('OPeNDAP url must be provided')
raise ValueError("OPeNDAP url must be provided")
if args.output_dir is None:
raise ValueError('MMD XML output directory must be provided')
raise ValueError("MMD XML output directory must be provided")

if not args.print_warnings:
warnings.filterwarnings("ignore")

assume_same_url_basename = False
if pathlib.Path(args.input).is_dir():
# Directory containing nc files
inputfiles = pathlib.Path(args.input).glob('*.nc')
inputfiles = pathlib.Path(args.input).glob("*.nc")
# If the input is a directory, we need to assume that the
# file and url basenames are the same
assume_same_url_basename = True
elif pathlib.Path(args.input).is_file():
# Single nc file
inputfiles = [args.input]
else:
raise ValueError(f'Invalid input: {args.input}')
raise ValueError(f"Invalid input: {args.input}")

url = None # dry-run option
ids = []
Expand All @@ -128,17 +132,20 @@ def main(args=None):
url = args.url
if "." in (pathlib.Path(file).stem):
infile = args.output_dir / pathlib.Path(file).stem
outfile = infile.with_suffix(infile.suffix+'.xml')
outfile = infile.with_suffix(infile.suffix+".xml")
else:
outfile = (args.output_dir / pathlib.Path(file).stem).with_suffix('.xml')
outfile = (args.output_dir / pathlib.Path(file).stem).with_suffix(".xml")
md = nc_to_mmd.Nc_to_mmd(str(file), opendap_url=url, output_file=outfile)
else:
md = nc_to_mmd.Nc_to_mmd(str(file), check_only=True)
overrides = None
if args.file_location is not None:
overrides = {"file_location": args.file_location}
mmd_yaml = yaml.load(
resource_string(md.__module__.split('.')[0], 'mmd_elements.yaml'),
resource_string(md.__module__.split(".")[0], "mmd_elements.yaml"),
Loader=yaml.FullLoader
)
metadata_id = md.get_metadata_identifier(mmd_yaml['metadata_identifier'],
metadata_id = md.get_metadata_identifier(mmd_yaml["metadata_identifier"],
netCDF4.Dataset(file))
if metadata_id in ids:
raise ValueError("Unique ID repetition. Please check your ID's.")
Expand All @@ -150,12 +157,13 @@ def main(args=None):
wms_layer_names=args.wms_layer_names,
checksum_calculation=args.checksum_calculation,
collection=args.collection,
parent=args.parent
parent=args.parent,
overrides=overrides
)
if args.log_ids:
with open(args.log_ids, 'a') as f:
with open(args.log_ids, "a") as f:
for mid in ids:
f.write(mid+'\n')
f.write(mid+"\n")


def _main(): # pragma: no cover
Expand All @@ -168,7 +176,7 @@ def _main(): # pragma: no cover
print(e)


if __name__ == '__main__': # pragma: no cover
if __name__ == "__main__": # pragma: no cover
try:
main(create_parser().parse_args())
except ValueError as e:
Expand Down
Loading
Loading