From d18c96859ce11451beba9f69ab4a305445e3a3dc Mon Sep 17 00:00:00 2001 From: brry Date: Thu, 30 May 2019 23:45:47 +0200 Subject: [PATCH] preparing CRAN release --- DESCRIPTION | 4 +- NEWS | 3 +- R/rdwd-package.R | 2 +- man/index.Rd | 1 + vignettes/cases.html | 49 +++++++------ vignettes/mapDWD.html | 10 +-- vignettes/rdwd.html | 166 ++++++++++++++++++++---------------------- 7 files changed, 116 insertions(+), 119 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 70974b7..d478fe3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: rdwd Title: Select and Download Climate Data from 'DWD' (German Weather Service) -Version: 1.0.30 -Date: 2019-05-26 +Version: 1.1.0 +Date: 2019-05-30 Depends: R(>= 2.10) SystemRequirements: Pandoc (>= 1.12.3), pandoc-citeproc Imports: berryFunctions (>= 1.17.0), pbapply diff --git a/NEWS b/NEWS index 3760aae..ccdc5bc 100644 --- a/NEWS +++ b/NEWS @@ -17,7 +17,7 @@ rdwd Version history ----------------------------- ----------------------------- -Version 1.1.0 (2019-05-) +Version 1.1.0 (2019-05-30) Highlights: - Binary Radolan data can now be read correctly - The new FTP server is used @@ -44,6 +44,7 @@ New user-visible functions + objects: - newColumnNames - dwdparams (renamed from parameter_abreviations) - gridbase +- gridIndex - projectRasterDWD - localtestdir (though mainly for internal usage) diff --git a/R/rdwd-package.R b/R/rdwd-package.R index b8fcead..7ed3f31 100644 --- a/R/rdwd-package.R +++ b/R/rdwd-package.R @@ -88,7 +88,7 @@ release_questions <- function() { #' \bold{gridIndex}: Vector of file paths at \code{\link{gridbase}}. #' #' @name index -#' @aliases fileIndex metaIndex geoIndex +#' @aliases fileIndex metaIndex geoIndex gridIndex #' @docType data #' @format #' \bold{fileIndex}: data.frame with character strings. ca 270k rows x 8 columns:\cr diff --git a/man/index.Rd b/man/index.Rd index 417ae2a..4db53c7 100644 --- a/man/index.Rd +++ b/man/index.Rd @@ -6,6 +6,7 @@ \alias{fileIndex} \alias{metaIndex} \alias{geoIndex} +\alias{gridIndex} \title{Indexes of files and metadata on the DWD CDC FTP server} \format{\bold{fileIndex}: data.frame with character strings. ca 270k rows x 8 columns:\cr \code{res}, \code{var}, \code{per} (see \code{\link{selectDWD}}), diff --git a/vignettes/cases.html b/vignettes/cases.html index 2c24adf..f61109f 100644 --- a/vignettes/cases.html +++ b/vignettes/cases.html @@ -12,7 +12,7 @@ - + rdwd use cases @@ -4179,7 +4179,7 @@

rdwd use cases

Berry Boessenkool,

-

2019-05-16

+

2019-05-30

@@ -4234,7 +4234,7 @@

Step 1: get the URLS of data to be downloaded

fileIndex$per=="historical" , "path" ] length(links) # 2001 elements - much better
-
## [1] 2140
+
## [1] 2142

Step 2: download the data

@@ -4247,10 +4247,11 @@

Step 2: download the data

Step 3: read the data

2k large datasets probably is way too much for memory, so we’ll use a custom reading function. It will only select the relevant time section and rainfall column. The latter will be named with the id extracted from the filename.

-
##           Par               Kurz          Einheit
-## RS         RS Niederschlagshoehe               mm
-## RSF       RSF  Niederschlagsform numerischer Code
-## SH_TAG SH_TAG        Schneehoehe               cm
+
##             Par               Kurz          Einheit
+## NSH_TAG NSH_TAG    Schneehoehe_Neu               cm
+## RS           RS Niederschlagshoehe               mm
+## RSF         RSF  Niederschlagsform numerischer Code
+## SH_TAG   SH_TAG        Schneehoehe               cm
-

+

Let’s see the locations of our stations in an interactive map.

@@ -4305,13 +4306,13 @@

step 4: visual data checks

leaflet(data=mygeoIndex) %>% addTiles() %>% addCircleMarkers(~lon, ~lat, popup=~display, stroke=T)
-
- +
+

For a static map with scaleBar, OSMscale works nicely but currently still has a Java dependency, see https://github.com/brry/OSMscale#installation

library(OSMscale)
 pointsMap("lat", "lon", mygeoIndex, fx=2, fy=1, pargs=list(lwd=3), 
                     col="blue", zoom=5)
-

+

TOC

@@ -4332,10 +4333,10 @@

1. Find meteo stations around a given point

head(m[,-14]) # don't show url column with long urls 
##   Stations_id von_datum bis_datum Stationshoehe geoBreite geoLaenge
 ## 1          NA        NA        NA            NA  49.21178  9.812475
-## 2        2848  19310101  20190512           458  49.25750  9.859400
-## 3        5988  20180523  20190513           415  49.16380  9.916200
-## 4        2787  20050101  20190513           354  49.24280  9.678600
-## 5        2787  19410101  20190513           354  49.24280  9.678600
+## 2        2848  19310101  20190527           458  49.25750  9.859400
+## 3        5988  20180523  20190527           415  49.16380  9.916200
+## 4        2787  20050101  20190527           354  49.24280  9.678600
+## 5        2787  19410101  20190527           354  49.24280  9.678600
 ## 6        5206  20081103  20161111           396  49.11750  9.897200
 ##                  Stationsname         Bundesland    res           var
 ## 1 Braunsbach catchment center               <NA>      z          <NA>
@@ -4357,8 +4358,8 @@ 
1. Find meteo stations around a given point

 leaflet(m) %>% addTiles() %>%
   addCircles(lng=9.812475, lat=49.211784, radius=30e3) %>%
   addCircleMarkers(~geoLaenge, ~geoBreite, col=~col, popup=~Stationsname)
-
- +
+

TOC

@@ -4464,7 +4465,7 @@

a) find available meteo stations for each district

# berryFunctions::colPointsLegend + sf plots = set margins, see note there! axis(1, line=-1); axis(2, line=-1, las=1) points(m[int[[2]], c("geoLaenge", "geoBreite")], pch=16, col=2, cex=1.8)
-

+

b) Average data per district

@@ -4507,12 +4508,12 @@

b) Average data per district

##   MESS_DATUM Hohenlohekreis Bodenseekreis Offenbach    Berlin
-## 1 2017-09-15          68.45          83.7        NA 34.333333
-## 2 2017-10-15          56.25          48.4        NA 84.183333
-## 3 2017-11-15         103.40          96.7        NA 69.616667
-## 4 2017-12-15          90.85          94.4        NA 39.866667
-## 5 2018-01-15         124.65         114.9        NA 67.816667
-## 6 2018-02-15          24.75          53.3        NA  3.183333
+## 1 2017-10-15 56.25 48.4 NA 84.183333 +## 2 2017-11-15 103.40 96.7 NA 69.616667 +## 3 2017-12-15 90.85 94.4 NA 39.866667 +## 4 2018-01-15 124.65 114.9 NA 67.816667 +## 5 2018-02-15 24.75 53.3 NA 3.183333 +## 6 2018-03-15 46.75 51.1 NA 47.700000

TOC

Any feedback on this package (or this vignette) is very welcome via github or !

diff --git a/vignettes/mapDWD.html b/vignettes/mapDWD.html index 7c5a8d8..03809c0 100644 --- a/vignettes/mapDWD.html +++ b/vignettes/mapDWD.html @@ -11,7 +11,7 @@ - + DWD weather stations map @@ -4177,7 +4177,7 @@

DWD weather stations map

-

2019-05-16

+

2019-05-30

@@ -4190,8 +4190,8 @@

2019-05-16

leaflet(geoIndex) %>% addTiles() %>% addCircles(~lon, ~lat, radius=900, stroke=F, color=~col)%>% addCircleMarkers(~lon, ~lat, popup=~display, stroke=F, color=~col) -
- +
+

Open this vignette locally for faster map reaction times than on the CRAN html view:

vignette("mapDWD")

The blue dots mark stations for which recent files are available (with >=1 file in ‘recent’ folder or ‘BIS_DATUM’ later than one year ago). The red dots mark all stations with only historical datasets.

@@ -4199,7 +4199,7 @@

2019-05-16

library(rdwd)  ;  data(geoIndex)  ;  library(leaflet) 
 leaflet(data=geoIndex[geoIndex$recentfile,]) %>% addTiles() %>%
         addCircleMarkers(~lon, ~lat, popup=~display, stroke=F)
-

To request the nonpublic datasets counted in the infobox, please contact . (The DWD cannot publish all datasets because of copyright restrictions).

+

To request the nonpublic datasets counted in the infobox, please contact or . (The DWD cannot publish all datasets because of copyright restrictions).

Note: geoIndex is created in the last section of rdwd-package.R.

diff --git a/vignettes/rdwd.html b/vignettes/rdwd.html index ef3e6d9..95b4ecb 100644 --- a/vignettes/rdwd.html +++ b/vignettes/rdwd.html @@ -12,7 +12,7 @@ - + rdwd: climate data from the German Weather Service @@ -303,7 +303,7 @@

rdwd: climate data from the German Weather Service

Berry Boessenkool,

-

2019-05-16

+

2019-05-30

@@ -328,7 +328,7 @@

2019-05-16

Intro

The R package rdwd contains code to select, download and read weather data from measuring stations across Germany. The German Weather Service (Deutscher Wetterdienst, DWD) provides thousands of datasets with weather observations through the FTP server online at

ftp://opendata.dwd.de/climate_environment/CDC/observations_germany/climate.

-

For data interpolated onto a 1 km raster, including radar data up to the last hour, see ftp://opendata.dwd.de/climate_environment/CDC/grids_germany/. Raster and binary reading functions are included since rdwd 1.1.0 (March 2019), although the latter doesn’t read the radolan binary files completely correct yet, see ?readDWD.binary.

+

For data interpolated onto a 1 km raster, including radar data up to the last hour, see ftp://opendata.dwd.de/climate_environment/CDC/grids_germany/. Raster and binary reading functions for the Radolan data are included in rdwd since May 2019.

For further details, please consult the DWD FTP server documentation
Please note that, as of June 2019, the old server at ftp://ftp-cdc.dwd.de will be phased out.

@@ -347,62 +347,61 @@

Package structure

Package installation

On Linux, instead of the last line above, use in the terminal (with lowercase rcurl):

sudo apt-get install r-cran-rcurl
-

If direct installation from CRAN doesn’t work, your R version might be too old. In that case it is really recommendable to update R. If you can’t update R, try installing from source (github) via instGit as mentioned above. If that’s not possible either, you might be able to source some functions from the package zip folder

- +

If direct installation from CRAN doesn’t work, your R version might be too old. In that case you’ll need to update R.

TOC

Basic usage

- + 
## 'data.frame':    550 obs. of  19 variables:
 ##  $ STATIONS_ID               : int  3987 3987 3987 3987 3987 3987 3987 3987 3987 3987 ...
-##  $ MESS_DATUM                : POSIXct, format: "2017-11-11" "2017-11-12" ...
+##  $ MESS_DATUM                : POSIXct, format: "2017-11-25" "2017-11-26" ...
 ##  $ QN_3                      : int  10 10 10 10 10 10 10 10 10 10 ...
-##  $ FX.Windspitze             : num  14.3 10.8 8.8 11.1 7.7 7.7 8.5 17.9 17.7 12.6 ...
-##  $ FM.Windgeschwindigkeit    : num  6.3 3.4 3.9 4 3.1 2.3 3.8 6.4 7.8 5.5 ...
+##  $ FX.Windspitze             : num  9.9 11.5 14.7 12.3 8.2 6 5.5 5.2 10.9 13.9 ...
+##  $ FM.Windgeschwindigkeit    : num  3.7 5.2 6 5.2 3.4 2.3 2.5 1.9 4.5 6 ...
 ##  $ QN_4                      : int  3 3 3 3 3 3 3 3 3 3 ...
-##  $ RSK.Niederschlagshoehe    : num  1 0.7 0 0 1 0.5 0 1.2 2.9 1.1 ...
-##  $ RSKF.Niederschlagsform    : int  6 6 6 6 6 6 0 6 6 6 ...
-##  $ SDK.Sonnenscheindauer     : num  0.483 1.15 0.583 0.9 0 ...
+##  $ RSK.Niederschlagshoehe    : num  16.1 0.2 3.6 0 0.2 0 0 0 0.8 2.1 ...
+##  $ RSKF.Niederschlagsform    : int  6 6 6 6 6 6 6 0 8 8 ...
+##  $ SDK.Sonnenscheindauer     : num  0 2.72 2.53 1.42 6.62 ...
 ##  $ SHK_TAG.Schneehoehe       : int  0 0 0 0 0 0 0 0 0 0 ...
-##  $ NM.Bedeckungsgrad         : num  6.8 6.2 6.4 5.5 8 7.7 3.7 6.1 5.8 6.5 ...
-##  $ VPM.Dampfdruck            : num  7.8 7.5 7.7 7.3 9.9 10.1 8 6.8 7.3 7.6 ...
-##  $ PM.Luftdruck              : num  994 991 1000 1010 1013 ...
-##  $ TMK.Lufttemperatur        : num  4.9 3.5 4 3.6 7.5 7.4 5.2 3.4 4 3.8 ...
-##  $ UPM.Relative_Feuchte      : num  90 95 94.7 91.3 95 ...
-##  $ TXK.Lufttemperatur_Max    : num  7.2 6.1 6.6 7.4 9 8.9 9.7 5.9 5.7 5.9 ...
-##  $ TNK.Lufttemperatur_Min    : num  3.3 1.1 0.9 -1 6.1 4.6 0.7 0.8 2 0.9 ...
-##  $ TGK.Lufttemperatur_5cm_min: num  1.7 -1.5 -2.7 -3.9 5.6 2.4 -2.1 -1.7 0.6 -1.2 ...
+##  $ NM.Bedeckungsgrad         : num  7.2 5.6 6.7 7.2 5.3 6 7.3 7.2 6 6 ...
+##  $ VPM.Dampfdruck            : num  8 6.8 7 7.7 6.3 6.1 6.2 6.6 5.7 7 ...
+##  $ PM.Luftdruck              : num  998 1005 1002 992 994 ...
+##  $ TMK.Lufttemperatur        : num  3.8 2.7 3.5 5.4 2.1 0.9 1 1.7 -0.4 2.9 ...
+##  $ UPM.Relative_Feuchte      : num  99.2 92.3 89.6 86 89 ...
+##  $ TXK.Lufttemperatur_Max    : num  5.5 5 5.2 6.9 5.6 4.2 3.5 3.1 2.4 5.1 ...
+##  $ TNK.Lufttemperatur_Min    : num  1.6 -0.1 0.9 3.8 -0.3 -1.3 -1.3 -1.8 -2.5 0.8 ...
+##  $ TGK.Lufttemperatur_5cm_min: num  -0.3 -2.2 -1.4 2.6 -2.2 -3.5 -3.2 -3.7 -5.3 -1.7 ...
 ##  $ eor                       : Factor w/ 1 level "eor": 1 1 1 1 1 1 1 1 1 1 ...

TOC

Plotting examples

Recent temperature time series:

- -

+ +

Long term climate graph:

- +

TOC

@@ -410,10 +409,10 @@

Plotting examples

Station selection

Weather stations can be selected geographically with the interactive map. All stations within a certain radius around a given lat-long position can be obtained with nearbyStations.

The DWD station IDs can be obtained from station names with

-
findID("Potsdam")
+
findID("Potsdam")
## Potsdam 
 ##    3987
-
findID("Koeln", exactmatch=FALSE)
+
findID("Koeln", exactmatch=FALSE)
## Warning: source -> withVisible -> eval -> eval -> createBerrysVignettes -
 ## > rmarkdown::render -> knitr::knit -> call_block -> block_exec -> in_dir -
 ## > evaluate -> evaluate::evaluate -> evaluate_call -> timing_fn -> handle -
@@ -432,68 +431,63 @@ 
Available files

 
 
 
If you find the file index to be outdated (Error in download.file … : cannot open URL), please let me know and I will update it. Meanwhile, use current=TRUE in selectDWD:

-
-
fileIndex is created with the function indexFTP used in the last section of rdwd-package.R.

-
+
+
fileIndex is created with the function indexFTP used in the last section of rdwd-package.R.

+
 
TOC

 
 

 
File selection

 
selectDWD is designed to be very flexible:

-
+
 
## [[1]]
 ## [1] "ftp://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/hourly/sun/historical/ stundenwerte_SD_03987_18930101_20181231_hist.zip"
 ## 
 ## [[2]]
 ## [1] "ftp://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/hourly/sun/historical/ stundenwerte_SD_05705_19510101_20181231_hist.zip"

 
If res/var/per are left NA, an interactive selection is opened with the available options for the given station.

-
+
 
## [1] "ftp://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/ daily/kl/recent/tageswerte_KL_03987_akt.zip"                       
 ## [2] "ftp://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/ daily/kl/historical/tageswerte_KL_03987_18930101_20181231_hist.zip"

 
There may be a differing number of available files for several stations across all folders. That’s why the default outvec is FALSE (unless per="hr").

-
+
 
## [[1]]
 ## [1] "s_germany/climate/annual/more_precip/historical/jahreswerte_RR_03467_19930101_20181231_hist.zip" 
-## [2] "s_germany/climate/annual/more_precip/historical/jahreswerte_RR_03467_19940101_20171231_hist.zip" 
-## [3] "s_germany/climate/annual/more_precip/historical/jahreswerte_RR_03467_19940101_20181231_hist.zip" 
-## [4] "s_germany/climate/annual/more_precip/recent/jahreswerte_RR_03467_akt.zip"                        
-## [5] "s_germany/climate/daily/more_precip/historical/tageswerte_RR_03467_19930601_20181231_hist.zip"   
-## [6] "s_germany/climate/daily/more_precip/recent/tageswerte_RR_03467_akt.zip"                          
-## [7] "s_germany/climate/monthly/more_precip/historical/monatswerte_RR_03467_19930601_20171231_hist.zip"
-## [8] "s_germany/climate/monthly/more_precip/historical/monatswerte_RR_03467_19930601_20181231_hist.zip"
-## [9] "s_germany/climate/monthly/more_precip/recent/monatswerte_RR_03467_akt.zip"                       
+## [2] "s_germany/climate/annual/more_precip/recent/jahreswerte_RR_03467_akt.zip"                        
+## [3] "s_germany/climate/daily/more_precip/historical/tageswerte_RR_03467_19930601_20181231_hist.zip"   
+## [4] "s_germany/climate/daily/more_precip/recent/tageswerte_RR_03467_akt.zip"                          
+## [5] "s_germany/climate/monthly/more_precip/historical/monatswerte_RR_03467_19930601_20171231_hist.zip"
+## [6] "s_germany/climate/monthly/more_precip/historical/monatswerte_RR_03467_19930601_20181231_hist.zip"
+## [7] "s_germany/climate/monthly/more_precip/recent/monatswerte_RR_03467_akt.zip"                       
 ## 
 ## [[2]]
 ## [1] "s_germany/climate/annual/more_precip/historical/jahreswerte_RR_05116_19920101_20061231_hist.zip" 
-## [2] "s_germany/climate/annual/more_precip/historical/jahreswerte_RR_05116_19930101_20051231_hist.zip" 
-## [3] "s_germany/climate/annual/more_precip/historical/jahreswerte_RR_05116_19930101_20061231_hist.zip" 
-## [4] "s_germany/climate/annual/more_precip/historical/jahreswerte_RR_05116_19930101_20181231_hist.zip" 
-## [5] "s_germany/climate/daily/more_precip/historical/tageswerte_RR_05116_19920701_20061231_hist.zip"   
-## [6] "s_germany/climate/monthly/more_precip/historical/monatswerte_RR_05116_19920701_20061231_hist.zip"

+## [2] "s_germany/climate/daily/more_precip/historical/tageswerte_RR_05116_19920701_20061231_hist.zip"   
+## [3] "s_germany/climate/monthly/more_precip/historical/monatswerte_RR_05116_19920701_20061231_hist.zip"

TOC

Metadata

selectDWD also uses a complete data.frame with meta information, metaIndex (derived from the “Beschreibung” files in fileIndex).

- -
## 'data.frame':    97102 obs. of  12 variables:
+
+
## 'data.frame':    96609 obs. of  12 variables:
 ##  $ Stations_id  : int  1 1 1 1 1 ...
 ##  $ von_datum    : int  18910101 18910101 18910101 18910101 19120101 ...
 ##  $ bis_datum    : int  19860630 19860630 19860630 19860630 19860630 ...
@@ -506,15 +500,15 @@ 
Metadata

 ##  $ var          : chr  "more_precip" "more_precip" ...
 ##  $ per          : chr  "historical" "recent" ...
 ##  $ hasfile      : logi  TRUE FALSE TRUE ...

-
+
 
dataDWD can download (and readDWD can correctly read) such a data.frame from any folder on the FTP server:

-
+
 
## [1] "ervations_germany/climate/monthly/more_precip/historical/RR_Monatswerte_Beschreibung_Stationen.txt"

-
-
Meta files may list stations for which there are actually no files. These refer to nonpublic datasets (The DWD cannot publish all datasets because of copyright restrictions). To request those, please contact .

+
+
Meta files may list stations for which there are actually no files. These refer to nonpublic datasets (The DWD cannot publish all datasets because of copyright restrictions). To request those, please contact  or .

 
TOC

 
Any feedback on this package (or this vignette) is very welcome via github or !