From fdab4b882a18348171757688c41729a5bfdffa7c Mon Sep 17 00:00:00 2001 From: Lance Albertson Date: Fri, 20 Oct 2017 16:08:20 -0700 Subject: [PATCH] Add script to import site images --- csvs/site_images.csv | 87 +++++++++++++++++++ data_scripts/get_all.sh | 3 +- data_scripts/get_photo_point_images.py | 2 +- data_scripts/get_site_images.py | 39 +++++++++ .../{pull-pp-images.sh => pull-files.sh} | 6 ++ data_scripts/update-csvs.sh | 1 + 6 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 csvs/site_images.csv create mode 100755 data_scripts/get_site_images.py rename data_scripts/{pull-pp-images.sh => pull-files.sh} (51%) diff --git a/csvs/site_images.csv b/csvs/site_images.csv new file mode 100644 index 000000000..9baa54ecd --- /dev/null +++ b/csvs/site_images.csv @@ -0,0 +1,87 @@ +"Site Name","Image" +"Adams Outdoor School","sites/streamwebs.org/files/images/site/adams.jpg" +"Agate Beach","sites/streamwebs.org/files/images/site/hammock-at-the-edge-of-the-world-may-2014.jpg" +"Amazon Creek & Fox Hollow","sites/streamwebs.org/files/images/site/photo.jpg" +"Amazon Creek @ Frank Kinney Park","sites/streamwebs.org/files/images/site/img_0443.jpg" +"Amazon Headwaters - Martin Street trail","sites/streamwebs.org/files/images/site/img_0121.jpg" +"Arrah Wanna Outdoor School","sites/streamwebs.org/files/images/site/arrah_wanna.jpg" +"Ash Creek at Talmadge MS","sites/streamwebs.org/files/images/site/ash_creek.jpg" +"Bald Hill Farm- Mulkey Creek","sites/streamwebs.org/files/images/site/mulkey_creek_trail.png" +"Bear Creek","sites/streamwebs.org/files/images/site/bear_creek_fall_08-_cs_004.jpg" +"Beaver Creek SNA Site 1","sites/streamwebs.org/files/images/site/201102_beavercreek_1.jpg" +"Beaver Creek SNA Site 2","sites/streamwebs.org/files/images/site/index.jpg" +"Beverly Beach","sites/streamwebs.org/files/images/site/splitting-headache.jpg" +"Blair Creek","sites/streamwebs.org/files/images/site/sonomacreek.jpg" +"Bottom Creek StreamBank Instream Wood","sites/streamwebs.org/files/images/site/bottom_bridge_resized.jpg" +"Brice Creek West Trailhead","sites/streamwebs.org/files/images/site/brice_mosby_creek_macro_nw_table_green_109.jpg" +"Camco, Inc. Restoration Project","sites/streamwebs.org/files/images/site/c01_7.jpg" +"Camp Creek","sites/streamwebs.org/files/images/site/oct_20_2008_008.jpg" +"Cave Creek (Lower/Covered Bridge)","sites/streamwebs.org/files/images/site/coveredbridge.jpg" +"Cave Creek (Upper)","sites/streamwebs.org/files/images/site/cave_creek_upper.jpg" +"Cave Entrance","sites/streamwebs.org/files/images/site/cave_entrance.jpg" +"CEC: Brown's Ferry Park","sites/streamwebs.org/files/images/site/feb_10_110.jpg" +"Clear Creek","sites/streamwebs.org/files/images/site/dsc_5471.jpg" +"Confluence of Battle Ax & Opal Creeks","sites/streamwebs.org/files/images/site/above_opal_pool-_randy_craig.jpg" +"Eagle Fern Park","sites/streamwebs.org/files/images/site/eaglefern_16.jpg" +"Enterprise City Park","sites/streamwebs.org/files/images/site/enterprise_fall_08_006.jpg" +"Enterprise Marr Pond Site","sites/streamwebs.org/files/images/site/photo_point_-_wet_wallowa_r_view_ds_from_upperend_survey_4-8-2008_022.jpg" +"Evergreen Elementary School Bioswale","sites/streamwebs.org/files/images/site/img_20131203_154026_744.jpg" +"Grande Ronde River","sites/streamwebs.org/files/images/site/wet.troy_school.10052010_001.jpg" +"Greasy Creek","sites/streamwebs.org/files/images/site/greasy_creek.jpg" +"High Lakes","sites/streamwebs.org/files/images/site/image294.jpg" +"HMSC Training Stream","sites/streamwebs.org/files/images/site/img_7422.jpg" +"Hoover's Dixon Creek Water Quality","sites/streamwebs.org/files/images/site/dixonathoover.jpg" +"Howard Outdoor School","sites/streamwebs.org/files/images/site/howard.jpg" +"Humbug Creek - Camp 18","sites/streamwebs.org/files/images/site/humbug_07.jpg" +"Imnaha River","sites/streamwebs.org/files/images/site/imnaha_fall_08_-_cs_012.jpg" +"Johnson Creek - NUHS","sites/streamwebs.org/files/images/site/johnson-creek-flood.jpg" +"Joseph City Park","sites/streamwebs.org/files/images/site/april_22.09_048.jpg" +"Joseph Prairie Creek","sites/streamwebs.org/files/images/site/october_1_2008_042.jpg" +"Knowles Creek Trip 5/17","sites/streamwebs.org/files/images/site/12_k_cr_20170519_091400_kc.jpg" +"LHS River Restoration Association; Had Irvine Park","sites/streamwebs.org/files/images/site/site_map.jpg" +"Lostine River Wolfe Ranch","sites/streamwebs.org/files/images/site/lostine_river_lower_end_ds_view_10212014.jpg" +"Lotus Isle Cove","sites/streamwebs.org/files/images/site/islandcove.jpg" +"Marys River","sites/streamwebs.org/files/images/site/marys_river.jpg" +"Matson Creek MWS Youth Program","sites/streamwebs.org/files/images/site/matson_creek_cleanup_013.jpg" +"Mingus Park","sites/streamwebs.org/files/images/site/night_at_mingus_017.jpg" +"Munsel Creek 18th St.","sites/streamwebs.org/files/images/site/photo_2.jpg" +"Namanu Outdoor School","sites/streamwebs.org/files/images/site/namanu.jpg" +"OC Brown Park - North Fork Deer Creek","sites/streamwebs.org/files/images/site/oc_brown_may_2016.jpg" +"OR Coast Aquarium","sites/streamwebs.org/files/images/site/oregon-coast-aquarium-shark-tube.jpg" +"Oregon 4-H Center","sites/streamwebs.org/files/images/site/566_oregon4hsign.jpg" +"Owyhee River, 8 miles below Owyhee Dam","sites/streamwebs.org/files/images/site/summer_2012_036.jpg" +"Pawn Trail Trip 5/17","sites/streamwebs.org/files/images/site/18_20170519_110904_pawn_1.jpg" +"Pony Creek MWS Youth Program","sites/streamwebs.org/files/images/site/cimg0043.jpg" +"Ramona Falls","sites/streamwebs.org/files/images/site/mariposa.jpg" +"REALMS Tumalo Creek Study Site","sites/streamwebs.org/files/images/site/100_0622_0.JPG" +"Reed Canyon","sites/streamwebs.org/files/images/site/215310_10150151507163335_4216609_n.jpg" +"River Restoration Association/Springbank Farm Survey","sites/streamwebs.org/files/images/site/dscn2386.jpg" +"Roaring Fork River Aspen, CO","sites/streamwebs.org/files/images/site/img_1835.jpg" +"Ryan Meadow Student Stewardship Projects","sites/streamwebs.org/files/images/site/img_8218.jpg" +"salmon river","sites/streamwebs.org/files/images/site/oneonta-gorge-waterfall.jpg" +"Salmon River #2","sites/streamwebs.org/files/images/site/dscf2181_0.jpg" +"Sandy HS AP Environmental Science","sites/streamwebs.org/files/images/site/pa191013.jpg" +"Sandy River Outdoor School","sites/streamwebs.org/files/images/site/sandy_river_0.jpg" +"Schooner Creek Restoration Large Wood Placement #8","sites/streamwebs.org/files/images/site/p1010717.jpg" +"SE Belmont","sites/streamwebs.org/files/images/site/20141226_135032.jpg" +"Seagrass Bed- HMSC","sites/streamwebs.org/files/images/site/img_2773.jpg" +"SHHS Biology Field Day","sites/streamwebs.org/files/images/site/dscn2713.jpg" +"Spitting Stone (Inside Cave)","sites/streamwebs.org/files/images/site/spitting_stone.jpg" +"Tenino Court Site","sites/streamwebs.org/files/images/site/cwa_139.jpg" +"Thompson's Mills State Heritage Site","sites/streamwebs.org/files/images/site/2.jpg" +"Toledo Elementary Stream","sites/streamwebs.org/files/images/site/toledo_elementary_stream.jpg" +"Tryon Creek - 1","sites/streamwebs.org/files/images/site/nuhs_tryon_creek.jpg" +"Tumalo Creek (REALMS Shevlin Park)","sites/streamwebs.org/files/images/site/DSCF2492.JPG" +"Viking Creek 9th St.","sites/streamwebs.org/files/images/site/photo_1.jpg" +"Viking Creek Rhody Dr","sites/streamwebs.org/files/images/site/vc_outlet_20170519_133625_vc_outlet.jpg" +"Viking Creek SHS","sites/streamwebs.org/files/images/site/img953910.jpg" +"Wallowa River Tamkaliks Site","sites/streamwebs.org/files/images/site/wallowa_river_fall_08-_cs_002.jpg" +"Wallowa River Wolfe Ranch","sites/streamwebs.org/files/images/site/wallowa_river_upper_end_ds_view_10212014.jpg" +"Wells Creek","sites/streamwebs.org/files/images/site/wells_creek.jpg" +"Wenaha River","sites/streamwebs.org/files/images/site/wet.troy_school.10052010_003.jpg" +"West Olalla Creek","sites/streamwebs.org/files/images/site/img_0020.jpg" +"Whalen Island","sites/streamwebs.org/files/images/site/562562_10200548649451299_588989842_n.jpg" +"Whychus Creek","sites/streamwebs.org/files/images/site/IMG_2125.JPG" +"WREN Camp Creek Site A","sites/streamwebs.org/files/images/site/wren_05092014-6495.jpg" +"WREN Camp Creek Site B","sites/streamwebs.org/files/images/site/wren_05092014-6490.jpg" +"Zigzag River","sites/streamwebs.org/files/images/site/zigzag_river.jpg" diff --git a/data_scripts/get_all.sh b/data_scripts/get_all.sh index b250b5c14..81ca65db9 100755 --- a/data_scripts/get_all.sh +++ b/data_scripts/get_all.sh @@ -14,11 +14,12 @@ set -e ./get_soil.py ./get_camera_points.py ./get_photo_points.py -./pull-pp-images.sh +./pull-files.sh # Run twice so that relations are created for renamed schools ./rename_schools.py ./rename_schools.py ./reset_dup_relations.py ./set_schools.py ./get_photo_point_images.py +./get_site_images.py rm -rf media/ diff --git a/data_scripts/get_photo_point_images.py b/data_scripts/get_photo_point_images.py index 861cacedd..5924c8a5c 100755 --- a/data_scripts/get_photo_point_images.py +++ b/data_scripts/get_photo_point_images.py @@ -36,7 +36,7 @@ photo_point = PhotoPoint.objects.get(id=row[1]) date = datetime.strptime(row[2], "%a, %Y-%m-%d") image_file = os.path.basename(row[3]) - # These files were pulled in via pull-pp-images.sh + # These files were pulled in via pull-files.sh image = open("../media/pp_photos/" + image_file, 'r') pp_image = PhotoPointImage.objects.update_or_create( diff --git a/data_scripts/get_site_images.py b/data_scripts/get_site_images.py new file mode 100755 index 000000000..c038428c4 --- /dev/null +++ b/data_scripts/get_site_images.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +import os +import sys +import csv + +from django.core.wsgi import get_wsgi_application +from django.core.files import File +# from django.contrib.gis.geos import GEOSGeometry + + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "streamwebs_frontend.settings") +# Set proj path to be relative to data_scripts directory +proj_path = "../streamwebs_frontend/" +sys.path.append(proj_path) +application = get_wsgi_application() + +from streamwebs.models import Site # NOQA + +if os.path.isdir("../sw_data/"): + datafile = '../sw_data/site_images.csv' +else: + datafile = '../csvs/site_images.csv' + + +# Site Name, Image +with open(datafile, 'r') as csvfile: + sitereader = csv.reader(csvfile) + for row in sitereader: + if row[0] != 'Site Name': # Skip the header + + image_file = os.path.basename(row[1]) + # These files were pulled in via pull-files.sh + image = open("../media/site_photos/" + image_file, 'r') + + site = Site.objects.get(site_name=row[0]) + site.image.save(image_file, File(image)) + site.save + +print "Site Images loaded." diff --git a/data_scripts/pull-pp-images.sh b/data_scripts/pull-files.sh similarity index 51% rename from data_scripts/pull-pp-images.sh rename to data_scripts/pull-files.sh index 92fdb2d34..39957b96d 100755 --- a/data_scripts/pull-pp-images.sh +++ b/data_scripts/pull-files.sh @@ -1,5 +1,11 @@ #!/bin/bash +DIR=$PWD mkdir -p ../media/pp_photos cd ../media/pp_photos wget -q -O- http://drupal.streamwebs.org/photo_point_image_files/csv | wget -nc -q -i- echo "Photo Point Images pulled from drupal site." +cd $DIR +mkdir -p ../media/site_photos +cd ../media/site_photos +wget -q -O- http://drupal.streamwebs.org/site_images | wget -nc -q -i- +echo "Site Images pulled from drupal site." diff --git a/data_scripts/update-csvs.sh b/data_scripts/update-csvs.sh index 2111db586..12f6405cf 100755 --- a/data_scripts/update-csvs.sh +++ b/data_scripts/update-csvs.sh @@ -32,5 +32,6 @@ curl http://streamwebs.org/active_schools_new/csv > active_schools.csv curl http://drupal.streamwebs.org/camera_points/csv > camera_points.csv curl http://drupal.streamwebs.org/photo_points/csv > photo_points.csv curl http://drupal.streamwebs.org/photo_point_images/csv > photo_point_images.csv +curl http://drupal.streamwebs.org/site_images.csv > site_images.csv sed -i '2s;^;"0","Unknown School","Elementary","2009-02-03 14:16","2009-02-03 14:16","1234 Unknown Street","Corvallis","Oregon","97330","United States"\n;' schools_info.csv