Merge pull request dailyerosion#113 from akrherz/dev220118

Omnibus

.github/workflows/main.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        PYTHON_VERSION: ["3.7", "3.8", "3.9"]
+        PYTHON_VERSION: ["3.8", "3.9", "3.10"]
     env:
       PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }}
     steps:

.pre-commit-config.yaml

@@ -3,4 +3,4 @@ repos:
     rev: 21.12b0
     hooks:
     - id: black
-      language_version: python3.8
+      language_version: python3.10

scripts/RT/queue_worker.py

@@ -2,6 +2,7 @@
 from functools import partial
 from concurrent.futures import ThreadPoolExecutor
 import re
+import os
 import socket
 import subprocess
 import sys
@@ -42,6 +43,7 @@ def run(channel, method, _props, rundata):
                 f"/i/{d['scenario']}/error/{d['huc8']}/{d['huc812']}/"
                 f"{d['huc12']}_{d['fpath']}.error"
             )
+            os.makedirs(os.path.dirname(errorfn), exist_ok=True)
             with open(errorfn, "wb") as fp:
                 hn = f"Hostname: {socket.gethostname()}\n"
                 fp.write(hn.encode("ascii"))

scripts/util/dump_ofe_results.py

@@ -31,8 +31,10 @@ def main():
             meta = read_sql(
                 """
                 with data as (
-                    select ofe, (max(elevation) - min(elevation)) /
-                    greatest(3, max(length) - min(length)) as slope,
+                    select ofe,
+                    (max(elevation) - min(elevation)) /
+                     greatest(3, max(length) - min(length)) as bulk_slope,
+                    max(slope) as max_slope,
                     max(genlu) as genlu,
                     max(landuse) as landuse, max(management) as management,
                     max(surgo) as surgo,
@@ -46,10 +48,13 @@ def main():
                 pgconn,
                 params=(huc12, fpath),
             )
+            # could have zeros from bulk_slope, so overwrite max value
+            meta.loc[meta["bulk_slope"] == 0, "bulk_slope"] = meta["max_slope"]
 
             for ofe in ofedf["ofe"].unique():
                 myofe = ofedf[ofedf["ofe"] == ofe]
                 meta_ofe = meta[meta["ofe"] == ofe]
+                print(meta_ofe)
                 if meta_ofe.empty:
                     print(ofe, huc12, fpath)
                     sys.exit()
@@ -61,7 +66,8 @@ def main():
                         "fpath": fpath,
                         "ofe": ofe,
                         "CropRotationString": meta_ofe["landuse"].values[0],
-                        "slope[1]": meta_ofe["slope"].values[0],
+                        "bulk_slope[1]": meta_ofe["bulk_slope"].values[0],
+                        "max_slope[1]": meta_ofe["max_slope"].values[0],
                         "soil_mukey": meta_ofe["surgo"].values[0],
                         "rainfall": -1,
                         "runoff[mm/yr]": myofe["runoff"].sum() / YEARS,