From 9f5cc596c75ca9ce8cb84e1d36b7d51d0bff000f Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 24 Jun 2021 15:17:29 -0600 Subject: [PATCH 1/3] new dockerfile, need to investigate testing --- .dockerignore | 8 +++- Dockerfile | 87 +++++++++++++--------------------------- docker-compose.debug.yml | 10 ----- docker-compose.yml | 11 ----- docker-entrypoint.sh | 20 --------- 5 files changed, 34 insertions(+), 102 deletions(-) delete mode 100644 docker-compose.debug.yml delete mode 100644 docker-compose.yml delete mode 100644 docker-entrypoint.sh diff --git a/.dockerignore b/.dockerignore index ffb2e60..3162949 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,6 @@ npm-debug.log Dockerfile* docker-compose* .dockerignore -.git .gitignore .env .venv @@ -11,4 +10,9 @@ docker-compose* */obj # README.md LICENSE -.vscode \ No newline at end of file +.vscode +build/ +dist/ +htmlcov/ +*.egg-info +.github diff --git a/Dockerfile b/Dockerfile index 4e4a008..1653ea1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,68 +1,37 @@ -# The purpose of this Docker image is to facilitate the download and -# conversion of the HRRR grib2 files. -FROM python:3.6.8-alpine3.9 +# Multistage build to build wgrib2 first +FROM python:3.8.10-buster as builder -# Install and make wgrib2 +WORKDIR /build/wgrib2 + +# build wgrib2 ENV CC gcc ENV FC gfortran -ENV CLASSPATH ".:/usr/local/bin/antlr.jar:$CLASSPATH" +RUN apt-get update \ + && apt-get install -y gfortran \ + && curl ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz | tar xvz \ + && cd /build/wgrib2/grib2 \ + && wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz \ + && wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-4.7.3.tar.gz \ + && sed -i "s/USE_NETCDF4=0/USE_NETCDF4=1/" makefile \ + && sed -i "s/USE_NETCDF3=1/USE_NETCDF3=0/" makefile \ + && make \ + && cp wgrib2/wgrib2 /usr/local/bin/wgrib2 + +############################################## +# main image +############################################## +FROM python:3.8.10-slim-buster + +COPY . /code WORKDIR /code -RUN apk --no-cache --virtual .build-dependencies add build-base curl gfortran cmake zlib-dev perl diffutils bash curl-dev m4 && \ - apk --no-cache add libgfortran libgomp libstdc++ && \ - curl ftp://ftp.cpc.ncep.noaa.gov/wd51we/wgrib2/wgrib2.tgz | tar xvz && \ - cd /code/grib2 && \ - wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.4/src/hdf5-1.10.4.tar.gz && \ - wget ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.6.1.tar.gz && \ - sed -i "s/USE_NETCDF4=0/USE_NETCDF4=1/" makefile && \ - sed -i "s/USE_NETCDF3=1/USE_NETCDF3=0/" makefile && \ - make && \ - cp wgrib2/wgrib2 /usr/local/bin/wgrib2 && \ - make deep-clean && \ - rm *.tar.gz - -# ncap2 needs antlr which is a java program -RUN apk add openjdk8-jre-base && \ - curl http://dust.ess.uci.edu/nco/antlr-2.7.7.tar.gz | tar xvz && \ - cd /code/antlr-2.7.7 && \ - export CLASSPATH=".:/usr/local/bin/antlr.jar:$CLASSPATH" && \ - ./configure --prefix=/usr/local --disable-examples && \ - make && \ - make test && \ - make install && \ - mv antlr.jar /usr/local/bin && \ - rm -rf /code/antlr-2.7.7 -# install NCO -# There is something wrong with either a dependency or with nco as the hdf5 headers are -# compiled with 1.10.4 but installed version is 1.10.5, go figure, just suppress the -# warnings and hope for the best! -ENV HDF5_DISABLE_VERSION_CHECK 2 -RUN echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \ - apk --no-cache --virtual .nco-dependencies add flex byacc && \ - apk --no-cache add netcdf netcdf-dev && \ - cd /code && \ - wget https://github.com/nco/nco/archive/4.7.2.tar.gz && \ - tar xvzf 4.7.2.tar.gz && \ - rm 4.7.2.tar.gz && \ - cd nco-4.7.2 && \ - ./configure --prefix=/usr/local && \ - make && \ - make install && \ - apk del .nco-dependencies && \ - rm -rf /code/nco-4.7.2 - -# Add the weather code -ADD . /code/weather_forecast_retrieval +COPY --from=builder /usr/local/bin/wgrib2 /usr/local/bin/wgrib2 # Add and build weather forecast retrival -RUN cd /code/weather_forecast_retrieval && \ - apk --no-cache add netcdf-utils hdf5 hdf5-dev libffi-dev && \ - CFLAGS="-g0 -Wl,--strip-all" \ - python3 -m pip install --no-cache-dir --compile --global-option=build_ext -r requirements.txt && \ - python3 setup.py install && \ - apk del .build-dependencies - -VOLUME /data +RUN apt-get update \ + && apt-get install -y git \ + && python3 -m pip install --no-cache-dir -r requirements.txt \ + && python3 setup.py install -# ENTRYPOINT ["/code/weather_forecast_retrieval/docker-entrypoint.sh"] \ No newline at end of file +VOLUME /data \ No newline at end of file diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml deleted file mode 100644 index ba74d08..0000000 --- a/docker-compose.debug.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '2.1' - -services: - weather_forecast_retrieval: - image: weather_forecast_retrieval - build: - context: . - dockerfile: Dockerfile - ports: - - 3000:3000 diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 9c06fbd..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: '3' - -services: - weather_forecast_retrieval: - image: weather_forecast_retrieval - build: . - user: "${CURRENT_UID}:1" - # command: run_hrrr_retrieval_dates - volumes: - - /data:/data - - /home/scotthavens/code/weather_forecast_retrieval/scripts:/code/config diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100644 index e4899be..0000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -set -e - -rhr='python3 /code/weather_forecast_retrieval/scripts/run_hrrr_retrieval' - -if [ $# -eq 0 ]; then - echo "Must pass config file to weather_forecast_retrieval" - exit 1 - -elif [[ "$1" == *.ini ]]; then - echo "Run weather_forecast_retrieval with config file" - umask 0002 - echo "$1" - exec $rhr "$1" - -else - echo "$@" - umask 0002 - exec "$@" -fi \ No newline at end of file From 436239ca9aa6f0b233afba7f654f9a95cb4d2265 Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Thu, 24 Jun 2021 16:18:47 -0600 Subject: [PATCH 2/3] dependancies --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1653ea1..1bdb845 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ COPY --from=builder /usr/local/bin/wgrib2 /usr/local/bin/wgrib2 # Add and build weather forecast retrival RUN apt-get update \ - && apt-get install -y git \ + && apt-get install -y git libeccodes-tools libgfortan5 libgomp1 \ && python3 -m pip install --no-cache-dir -r requirements.txt \ && python3 setup.py install From a5b29edbeb4757bb6dbbab9c834f54341f70a38e Mon Sep 17 00:00:00 2001 From: Scott Havens Date: Fri, 25 Jun 2021 09:35:32 -0600 Subject: [PATCH 3/3] updated docker image and have tested that everything works except grib2nc, updated readme --- Dockerfile | 9 +++++---- README.md | 15 +++++---------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1bdb845..85e3744 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.8.10-buster as builder WORKDIR /build/wgrib2 -# build wgrib2 +# build wgrib2, this takes a while ENV CC gcc ENV FC gfortran @@ -29,9 +29,10 @@ WORKDIR /code COPY --from=builder /usr/local/bin/wgrib2 /usr/local/bin/wgrib2 # Add and build weather forecast retrival -RUN apt-get update \ - && apt-get install -y git libeccodes-tools libgfortan5 libgomp1 \ +RUN apt-get update -y \ + && apt-get install -y git libeccodes-tools libgfortran5 libgomp1 \ && python3 -m pip install --no-cache-dir -r requirements.txt \ - && python3 setup.py install + && python3 setup.py install \ + && rm -rf /var/lib/apt/lists/* VOLUME /data \ No newline at end of file diff --git a/README.md b/README.md index 50c2366..58f4670 100755 --- a/README.md +++ b/README.md @@ -35,17 +35,16 @@ ln wgrib2/wgrib2 ~/bin/wgrib2 ## Docker -The retrieval aspect of `weather_forecast_retieval` has been built into a Docker image based on the Python 3 Alpine linux image. This allows for a docker deployment to run and retrieve HRRR data and convert to netcdf if needed. To use, first build the image +The retrieval aspect of `weather_forecast_retieval` has been built into a Docker image based Python 3.8. This allows for a docker deployment to run and retrieve HRRR data. The docker image can call any of the command line programs in `weather_forecast_retrieval`. + +For example, to run `hrrr_nomads` with docker: ``` -docker build -t usdaarsnwrc/weather_forecast_retieval . +docker run --rm usdaarsnwrs/weather_forecast_retrieval hrrr_nomads -l 3 -f 0,1,2 --bbox="-119,-118,37,38" -o /path/to/output -p /path/to/crop/output ``` -Grab a coffee as this has to compile `pandas` from source (10+ minutes of compile time). Once completed, modify or create a new `docker-compose.yml` and modify the volume attachments as necessary. There are 2 volumes to attach, a `data` drive mounted to `/data` and the config file folders at `/code/config`. To setup the download, the config file is passed to `docker-compose`: +The paths to the output directories are internal to the docker image and the necessary volume mounts are needed. -``` -docker-compose run weather_forecast_retrieval /code/config/hrrr.ini -``` # Command line usage @@ -170,7 +169,3 @@ optional arguments: --verbose increase logging verbosity --overwrite Download and overwrite existing HRRR files ``` - -## convert_grib2nc - -## run_hrrr_retrieval