From cc26cabbd298e496ea118b3667e79e0be3ad976b Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 7 Apr 2022 17:36:35 +0545 Subject: [PATCH 01/69] remove absolute version of libraries --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 05bbbcca9..c607dd209 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,6 @@ gunicorn==19.9.0 osmium~=2.15.2 dramatiq[redis,watch]~=1.7.0 boto3~=1.9.238 # needed for cloudwatch -rasterio~=1.0.25 +rasterio osm-export-tool==0.0.25 -rtree==0.9.1 +rtree From 76e75eaf3658be5706e2171562370aa00d88c1ec Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 7 Apr 2022 21:57:49 +0545 Subject: [PATCH 02/69] added temp test api call --- .gitignore | 1 + tasks/models.py | 12 ++----- tasks/task_runners.py | 73 +++++++++++++++++++++++++++++++++---------- ui/yarn.lock | 17 ++++++++++ 4 files changed, 77 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 33d45f659..b62a4c39c 100755 --- a/.gitignore +++ b/.gitignore @@ -84,5 +84,6 @@ core/settings/secret.py .env node_modules ui/static/ui/js/bundle.js +ui/static/ui/js/ *.js.map secrets diff --git a/tasks/models.py b/tasks/models.py index a09e308d2..16ffb2fc9 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -87,17 +87,11 @@ def duration(self): @property def download_urls(self): def fdownload(fname): - try: - filesize_bytes = os.path.getsize(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid), fname).encode('utf-8')) - except Exception: - filesize_bytes = 0 - - download_url = os.path.join(settings.EXPORT_MEDIA_ROOT,str(self.run.uid), fname) return { "filename":fname, - "filesize_bytes": filesize_bytes, - "download_url":download_url, - "absolute_download_url":settings.HOSTNAME + download_url + "filesize_bytes": self.filesize_bytes, + "download_url":self.filenames, + "absolute_download_url":self.filenames } return map(fdownload, self.filenames) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 4fd420fe8..7ea07e62a 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -16,7 +16,7 @@ if not apps.ready and not settings.configured: django.setup() - +import requests import dramatiq from raven import Client @@ -34,8 +34,9 @@ from osm_export_tool.geometry import load_geometry from osm_export_tool.sources import Overpass, OsmiumTool from osm_export_tool.package import create_package, create_posm_bundle - +import json import shapely.geometry +import shapely from .email import ( send_completion_notification, @@ -85,8 +86,12 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - run_task_async_ondemand.send(run_uid) + print(" on demand") + run_task_remote(run_uid) + db.close_old_connections() + # run_task_async_ondemand.send(run_uid) else: + run_task_async_scheduled.send(run_uid) return run @@ -101,6 +106,8 @@ def run_task_async_scheduled(run_uid): db.close_old_connections() def run_task_remote(run_uid): + print("runtaskondemand") + try: run = ExportRun.objects.get(uid=run_uid) run.status = 'RUNNING' @@ -143,18 +150,20 @@ def start_task(name): task.started_at = timezone.now() task.save() - def finish_task(name,created_files,planet_file=False): + def finish_task(name,response_back,planet_file=False): LOG.debug('Task Finish: {0} for run: {1}'.format(name, run_uid)) task = ExportTask.objects.get(run__uid=run_uid, name=name) task.status = 'SUCCESS' task.finished_at = timezone.now() # assumes each file only has one part (all are zips or PBFs) - task.filenames = [basename(file.parts[0]) for file in created_files] + print("saved file name") + # print([basename(file.parts[0]) for file in created_files]) + task.filenames = [response_back['download_url']] if planet_file is False: - total_bytes = 0 - for file in created_files: - total_bytes += file.size() - task.filesize_bytes = total_bytes + # total_bytes = 0 + # for file in created_files: + # total_bytes += file.size() + task.filesize_bytes = response_back['zip_file_size'] task.save() is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists() @@ -330,17 +339,41 @@ def add_metadata(z,theme): h = tabular.Handler(tabular_outputs,mapping,polygon_centroid=polygon_centroid) source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir, mapping=mapping) else: - h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) - mapping_filter = mapping - if job.unfiltered: - mapping_filter = None - source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) + # print(mapping) + geom_json=shapely.geometry.mapping(geom) + + + # defining the api-endpoint + API_ENDPOINT = "http://18.209.245.110:8000/raw-data/current-snapshot/" + + + + # data to be sent to api + data = {"geometry":geom_json} + + + # sending post request and saving response as response object + headers = {'Content-type': "text/plain; charset=utf-8"} + r = requests.post(url = API_ENDPOINT, data = json.dumps(data) ,headers=headers) + + # extracting response text + print(r) + response_back = r.json() + # print("Response is:%s"%response_back) + # print(job) + # h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) + # mapping_filter = mapping + # if job.unfiltered: + # mapping_filter = None + # print("came here ") + # source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) LOG.debug('Source start for run: {0}'.format(run_uid)) - source_path = source.path() + # print("again came here") + # source_path = source.path() LOG.debug('Source end for run: {0}'.format(run_uid)) - h.apply_file(source_path, locations=True, idx='sparse_file_array') + # h.apply_file(source_path, locations=True, idx='sparse_file_array') bundle_files = [] @@ -351,10 +384,16 @@ def add_metadata(z,theme): finish_task('geopackage',[zipped]) if shp: + + start_task('shp') + shp.finalize() zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) bundle_files += shp.files - finish_task('shp',[zipped]) + print("zippedfinal") + print([zipped]) + print(response_back) + finish_task('shp',response_back) if kml: kml.finalize() diff --git a/ui/yarn.lock b/ui/yarn.lock index e2e3b9678..2ff387048 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -38,6 +38,13 @@ react-day-picker "^5.3.0" tslib "^1.5.0" +"@hotosm/iso-countries-languages@^1.0.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@hotosm/iso-countries-languages/-/iso-countries-languages-1.1.2.tgz#06d751c36f787dae2e172650657e9bcea3cf6f0f" + integrity sha512-BmjgzRGwKt0eaUNTNo3LghX5gaa0jVJUbcA4t8iKesvOotoWJiioGvIcWnFTiNBKeEh+0TibFWqUuJx1kmIo3g== + dependencies: + prettier "^2.0.5" + "@mapbox/geojson-area@^0.2.2": version "0.2.2" resolved "https://registry.yarnpkg.com/@mapbox/geojson-area/-/geojson-area-0.2.2.tgz#18d7814aa36bf23fbbcc379f8e26a22927debf10" @@ -1630,6 +1637,11 @@ core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" +country-iso-3-to-2@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/country-iso-3-to-2/-/country-iso-3-to-2-1.1.0.tgz#509bc7a4d2de9abfc57ad2d18fd8a40b85940a75" + integrity sha512-1+N29RRXHIsNrdxFYz/xqB6eacm592SQBExLBjHpr+nrLsKiikXvrFYM0RgW6uJBJgnvcqQH64ukydPF8ShBEA== + create-ecdh@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.0.tgz#888c723596cdf7612f6498233eebd7a35301737d" @@ -4427,6 +4439,11 @@ prettier@^1.5.3: version "1.7.4" resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.7.4.tgz#5e8624ae9363c80f95ec644584ecdf55d74f93fa" +prettier@^2.0.5: + version "2.6.2" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.6.2.tgz#e26d71a18a74c3d0f0597f55f01fb6c06c206032" + integrity sha512-PkUpF+qoXTqhOeWL9fu7As8LXsIUZ1WYaJiY/a7McAQzxjk82OF0tibkFXVCDImZtWxbvojFjerkiLb0/q8mew== + private@^0.1.6, private@^0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/private/-/private-0.1.7.tgz#68ce5e8a1ef0a23bb570cc28537b5332aba63ef1" From c1fc8912e7df1072e96ac6f7152c5e59d2430ee1 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 14 Apr 2022 19:03:56 +0545 Subject: [PATCH 03/69] added Galaxy class call function --- core/settings/project.py | 4 + doc/setup-development.md | 3 + requirements.txt | 2 +- tasks/task_runners.py | 10 +- ui/yarn.lock | 332 +++++---------------------------------- 5 files changed, 50 insertions(+), 301 deletions(-) diff --git a/core/settings/project.py b/core/settings/project.py index 204e8b320..d052ee9a5 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -66,6 +66,10 @@ # url to overpass api endpoint OVERPASS_API_URL = os.getenv('OVERPASS_API_URL', 'http://overpass-api.de/api/') + +#url to galaxy api endpoint +GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://127.0.0.1:8080/raw-data/current-snapshot/') + GENERATE_MWM = os.getenv('GENERATE_MWM','/usr/local/bin/generate_mwm.sh') GENERATOR_TOOL = os.getenv('GENERATOR_TOOL','/usr/local/bin/generator_tool') PLANET_FILE = os.getenv('PLANET_FILE','') diff --git a/doc/setup-development.md b/doc/setup-development.md index 1597afb82..f6015ba07 100644 --- a/doc/setup-development.md +++ b/doc/setup-development.md @@ -110,6 +110,9 @@ Most of these environment variables have reasonable default settings. * `OSMAND_MAP_CREATOR_DIR` path to directory where OsmAndMapCreator is installed * `GARMIN_CONFIG`, `GARMIN_MKGMAP` absolute paths to garmin JARs * `OVERPASS_API_URL` url of Overpass api endpoint + +* `GALAXY_API_URL` url of Galaxy api endpoint + * `DATABASE_URL` Database URL. Defaults to `postgres:///exports` * `DEBUG` Whether to enable debug mode. Defaults to `False` (production). * `DJANGO_ENV` Django environment. Set to `development` to enable development tools and email logging to console. diff --git a/requirements.txt b/requirements.txt index c607dd209..c62e61dcd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ djangorestframework~=3.6.3 hdx-python-api==2.1.5 mercantile~=0.10.0 psycopg2==2.8.3 -python-openid==2.2.5 +python-openid social-auth-app-django==2.1.0 social-auth-core==1.7.0 pytz diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 7ea07e62a..7175a13ce 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -32,7 +32,7 @@ import osm_export_tool.nontabular as nontabular from osm_export_tool.mapping import Mapping from osm_export_tool.geometry import load_geometry -from osm_export_tool.sources import Overpass, OsmiumTool +from osm_export_tool.sources import Overpass, OsmiumTool , Galaxy from osm_export_tool.package import create_package, create_posm_bundle import json import shapely.geometry @@ -344,7 +344,7 @@ def add_metadata(z,theme): # defining the api-endpoint - API_ENDPOINT = "http://18.209.245.110:8000/raw-data/current-snapshot/" + API_ENDPOINT = settings.GALAXY_API_URL @@ -362,12 +362,12 @@ def add_metadata(z,theme): # print("Response is:%s"%response_back) # print(job) # h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) - # mapping_filter = mapping + mapping_filter = mapping # if job.unfiltered: # mapping_filter = None # print("came here ") - # source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) - + source = Galaxy(settings.GALAXY_API_URL,geom,use_curl=True,mapping=mapping_filter) + source.fetch() LOG.debug('Source start for run: {0}'.format(run_uid)) # print("again came here") # source_path = source.path() diff --git a/ui/yarn.lock b/ui/yarn.lock index 2ff387048..95e870203 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -1,8 +1,10 @@ # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. # yarn lockfile v1 +# yarn v0.25.2 +# node v10.13.0 -"@blueprintjs/core@^1.24.0": +"@blueprintjs/core@^1.24.0", "@blueprintjs/core@^1.3.0": version "1.30.0" resolved "https://registry.yarnpkg.com/@blueprintjs/core/-/core-1.30.0.tgz#ae1532e57757ce079cf0cc2071981cc49999011e" dependencies: @@ -15,19 +17,6 @@ tether "^1.4" tslib "^1.5.0" -"@blueprintjs/core@^1.3.0": - version "1.24.0" - resolved "https://registry.yarnpkg.com/@blueprintjs/core/-/core-1.24.0.tgz#879cf0b17d93c871c223ed478343cdda1abe45d2" - dependencies: - "@types/dom4" "^1.5.20" - "@types/tether" "^1.1.27" - classnames "^2.2" - dom4 "^1.8" - normalize.css "4.1.1" - pure-render-decorator "^1.1" - tether "^1.4" - tslib "^1.5.0" - "@blueprintjs/datetime@^1.19.0": version "1.22.0" resolved "https://registry.yarnpkg.com/@blueprintjs/datetime/-/datetime-1.22.0.tgz#a62f6234ffe50526f8bec557ee31327c9488f852" @@ -41,7 +30,6 @@ "@hotosm/iso-countries-languages@^1.0.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@hotosm/iso-countries-languages/-/iso-countries-languages-1.1.2.tgz#06d751c36f787dae2e172650657e9bcea3cf6f0f" - integrity sha512-BmjgzRGwKt0eaUNTNo3LghX5gaa0jVJUbcA4t8iKesvOotoWJiioGvIcWnFTiNBKeEh+0TibFWqUuJx1kmIo3g== dependencies: prettier "^2.0.5" @@ -333,20 +321,11 @@ aws4@^1.2.1: axios@^0.18.1: version "0.18.1" resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.1.tgz#ff3f0de2e7b5d180e757ad98000f1081b87bcea3" - integrity sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g== dependencies: follow-redirects "1.5.10" is-buffer "^2.0.2" -babel-code-frame@^6.11.0, babel-code-frame@^6.16.0, babel-code-frame@^6.22.0: - version "6.22.0" - resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.22.0.tgz#027620bee567a88c32561574e7fd0801d33118e4" - dependencies: - chalk "^1.1.0" - esutils "^2.0.2" - js-tokens "^3.0.0" - -babel-code-frame@^6.26.0: +babel-code-frame@^6.11.0, babel-code-frame@^6.16.0, babel-code-frame@^6.22.0, babel-code-frame@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" dependencies: @@ -1012,31 +991,14 @@ babel-register@^6.26.0: mkdirp "^0.5.1" source-map-support "^0.4.15" -babel-runtime@^6.11.6, babel-runtime@^6.18.0, babel-runtime@^6.2.0, babel-runtime@^6.22.0: - version "6.23.0" - resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.23.0.tgz#0a9489f144de70efb3ce4300accdb329e2fc543b" - dependencies: - core-js "^2.4.0" - regenerator-runtime "^0.10.0" - -babel-runtime@^6.26.0: +babel-runtime@^6.11.6, babel-runtime@^6.18.0, babel-runtime@^6.2.0, babel-runtime@^6.22.0, babel-runtime@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" dependencies: core-js "^2.4.0" regenerator-runtime "^0.11.0" -babel-template@^6.24.1: - version "6.25.0" - resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.25.0.tgz#665241166b7c2aa4c619d71e192969552b10c071" - dependencies: - babel-runtime "^6.22.0" - babel-traverse "^6.25.0" - babel-types "^6.25.0" - babylon "^6.17.2" - lodash "^4.2.0" - -babel-template@^6.26.0: +babel-template@^6.24.1, babel-template@^6.26.0, babel-template@^6.7.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.26.0.tgz#de03e2d16396b069f46dd9fff8521fb1a0e35e02" dependencies: @@ -1046,45 +1008,7 @@ babel-template@^6.26.0: babylon "^6.18.0" lodash "^4.17.4" -babel-template@^6.7.0: - version "6.23.0" - resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.23.0.tgz#04d4f270adbb3aa704a8143ae26faa529238e638" - dependencies: - babel-runtime "^6.22.0" - babel-traverse "^6.23.0" - babel-types "^6.23.0" - babylon "^6.11.0" - lodash "^4.2.0" - -babel-traverse@^6.23.0, babel-traverse@^6.23.1: - version "6.23.1" - resolved "https://registry.yarnpkg.com/babel-traverse/-/babel-traverse-6.23.1.tgz#d3cb59010ecd06a97d81310065f966b699e14f48" - dependencies: - babel-code-frame "^6.22.0" - babel-messages "^6.23.0" - babel-runtime "^6.22.0" - babel-types "^6.23.0" - babylon "^6.15.0" - debug "^2.2.0" - globals "^9.0.0" - invariant "^2.2.0" - lodash "^4.2.0" - -babel-traverse@^6.24.1, babel-traverse@^6.25.0: - version "6.25.0" - resolved "https://registry.yarnpkg.com/babel-traverse/-/babel-traverse-6.25.0.tgz#2257497e2fcd19b89edc13c4c91381f9512496f1" - dependencies: - babel-code-frame "^6.22.0" - babel-messages "^6.23.0" - babel-runtime "^6.22.0" - babel-types "^6.25.0" - babylon "^6.17.2" - debug "^2.2.0" - globals "^9.0.0" - invariant "^2.2.0" - lodash "^4.2.0" - -babel-traverse@^6.26.0: +babel-traverse@^6.23.1, babel-traverse@^6.24.1, babel-traverse@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-traverse/-/babel-traverse-6.26.0.tgz#46a9cbd7edcc62c8e5c064e2d2d8d0f4035766ee" dependencies: @@ -1098,25 +1022,7 @@ babel-traverse@^6.26.0: invariant "^2.2.2" lodash "^4.17.4" -babel-types@^6.19.0, babel-types@^6.24.1, babel-types@^6.25.0: - version "6.25.0" - resolved "https://registry.yarnpkg.com/babel-types/-/babel-types-6.25.0.tgz#70afb248d5660e5d18f811d91c8303b54134a18e" - dependencies: - babel-runtime "^6.22.0" - esutils "^2.0.2" - lodash "^4.2.0" - to-fast-properties "^1.0.1" - -babel-types@^6.23.0: - version "6.23.0" - resolved "https://registry.yarnpkg.com/babel-types/-/babel-types-6.23.0.tgz#bb17179d7538bad38cd0c9e115d340f77e7e9acf" - dependencies: - babel-runtime "^6.22.0" - esutils "^2.0.2" - lodash "^4.2.0" - to-fast-properties "^1.0.1" - -babel-types@^6.26.0: +babel-types@^6.19.0, babel-types@^6.23.0, babel-types@^6.24.1, babel-types@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-types/-/babel-types-6.26.0.tgz#a3b073f94ab49eb6fa55cd65227a334380632497" dependencies: @@ -1125,19 +1031,11 @@ babel-types@^6.26.0: lodash "^4.17.4" to-fast-properties "^1.0.3" -babylon@^6.11.0, babylon@^6.15.0: - version "6.16.1" - resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.16.1.tgz#30c5a22f481978a9e7f8cdfdf496b11d94b404d3" - -babylon@^6.17.0, babylon@^6.17.2: - version "6.17.4" - resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.17.4.tgz#3e8b7402b88d22c3423e137a1577883b15ff869a" - -babylon@^6.18.0: +babylon@^6.17.0, babylon@^6.18.0: version "6.18.0" resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.18.0.tgz#af2f3b88fa6f5c1e4c634d1a0f8eac4f55b395e3" -balanced-match@^0.4.1, balanced-match@^0.4.2: +balanced-match@^0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" @@ -1201,14 +1099,6 @@ boom@2.x.x: bootstrap@~3.4.1: version "3.4.1" resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.4.1.tgz#c3a347d419e289ad11f4033e3c4132b87c081d72" - integrity sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA== - -brace-expansion@^1.0.0: - version "1.1.6" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.6.tgz#7197d7eaa9b87e648390ea61fc66c84427420df9" - dependencies: - balanced-match "^0.4.1" - concat-map "0.0.1" brace-expansion@^1.1.7: version "1.1.8" @@ -1302,10 +1192,6 @@ buffer-indexof@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/buffer-indexof/-/buffer-indexof-1.1.0.tgz#f54f647c4f4e25228baa656a2e57e43d5f270982" -buffer-shims@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" - buffer-xor@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" @@ -1395,7 +1281,7 @@ chain-function@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/chain-function/-/chain-function-1.0.0.tgz#0d4ab37e7e18ead0bdc47b920764118ce58733dc" -chalk@^1.0.0, chalk@^1.1.0, chalk@^1.1.1, chalk@^1.1.3: +chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" dependencies: @@ -1625,11 +1511,7 @@ core-js@^1.0.0, core-js@^1.1.1: version "1.2.7" resolved "https://registry.yarnpkg.com/core-js/-/core-js-1.2.7.tgz#652294c14651db28fa93bd2d5ff2983a4f08c636" -core-js@^2.4.0: - version "2.4.1" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.4.1.tgz#4de911e667b0eae9124e34254b53aea6fc618d3e" - -core-js@^2.5.0: +core-js@^2.4.0, core-js@^2.5.0: version "2.5.1" resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.1.tgz#ae6874dc66937789b80754ff5428df66819ca50b" @@ -1640,7 +1522,6 @@ core-util-is@~1.0.0: country-iso-3-to-2@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/country-iso-3-to-2/-/country-iso-3-to-2-1.1.0.tgz#509bc7a4d2de9abfc57ad2d18fd8a40b85940a75" - integrity sha512-1+N29RRXHIsNrdxFYz/xqB6eacm592SQBExLBjHpr+nrLsKiikXvrFYM0RgW6uJBJgnvcqQH64ukydPF8ShBEA== create-ecdh@^4.0.0: version "4.0.0" @@ -1669,15 +1550,7 @@ create-hmac@^1.1.0, create-hmac@^1.1.2, create-hmac@^1.1.4: safe-buffer "^5.0.1" sha.js "^2.4.8" -create-react-class@^15.5.2: - version "15.6.0" - resolved "https://registry.yarnpkg.com/create-react-class/-/create-react-class-15.6.0.tgz#ab448497c26566e1e29413e883207d57cfe7bed4" - dependencies: - fbjs "^0.8.9" - loose-envify "^1.3.1" - object-assign "^4.1.1" - -create-react-class@^15.6.0: +create-react-class@^15.5.2, create-react-class@^15.6.0: version "15.6.2" resolved "https://registry.yarnpkg.com/create-react-class/-/create-react-class-15.6.2.tgz#cf1ed15f12aad7f14ef5f2dfe05e6c42f91ef02a" dependencies: @@ -1845,7 +1718,7 @@ debug@2.6.7: dependencies: ms "2.0.0" -debug@2.6.8, debug@^2.6.6, debug@^2.6.8: +debug@2.6.8, debug@^2.1.1, debug@^2.2.0, debug@^2.6.6, debug@^2.6.8: version "2.6.8" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" dependencies: @@ -1854,16 +1727,9 @@ debug@2.6.8, debug@^2.6.6, debug@^2.6.8: debug@=3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== dependencies: ms "2.0.0" -debug@^2.1.1, debug@^2.2.0: - version "2.6.3" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.3.tgz#0f7eb8c30965ec08c72accfa0130c8b79984141d" - dependencies: - ms "0.7.2" - debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" @@ -2051,11 +1917,7 @@ ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" -electron-to-chromium@^1.2.7: - version "1.2.8" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.2.8.tgz#22c2e6200d350da27d6050db7e3f6f85d18cf4ed" - -electron-to-chromium@^1.3.14: +electron-to-chromium@^1.2.7, electron-to-chromium@^1.3.14: version "1.3.15" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.15.tgz#08397934891cbcfaebbd18b82a95b5a481138369" @@ -2325,10 +2187,6 @@ esprima@^2.6.0: version "2.7.3" resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" -esprima@^3.1.1: - version "3.1.3" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" - esprima@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" @@ -2488,19 +2346,7 @@ faye-websocket@~0.11.0: dependencies: websocket-driver ">=0.5.1" -fbjs@^0.8.0: - version "0.8.14" - resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-0.8.14.tgz#d1dbe2be254c35a91e09f31f9cd50a40b2a0ed1c" - dependencies: - core-js "^1.0.0" - isomorphic-fetch "^2.1.1" - loose-envify "^1.0.0" - object-assign "^4.1.0" - promise "^7.1.1" - setimmediate "^1.0.5" - ua-parser-js "^0.7.9" - -fbjs@^0.8.16: +fbjs@^0.8.0, fbjs@^0.8.16, fbjs@^0.8.6: version "0.8.16" resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-0.8.16.tgz#5e67432f550dc41b572bf55847b8aca64e5337db" dependencies: @@ -2512,7 +2358,7 @@ fbjs@^0.8.16: setimmediate "^1.0.5" ua-parser-js "^0.7.9" -fbjs@^0.8.6, fbjs@^0.8.9: +fbjs@^0.8.9: version "0.8.11" resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-0.8.11.tgz#340b590b8a2278a01ef7467c07a16da9b753db24" dependencies: @@ -2611,7 +2457,6 @@ flatten@^1.0.2: follow-redirects@1.5.10: version "1.5.10" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" - integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== dependencies: debug "=3.1.0" @@ -2745,7 +2590,7 @@ glob-parent@^2.0.0: dependencies: is-glob "^2.0.0" -glob@^7.0.0, glob@^7.1.2: +glob@^7.0.0, glob@^7.0.5, glob@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: @@ -2756,7 +2601,7 @@ glob@^7.0.0, glob@^7.1.2: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.3, glob@^7.0.5: +glob@^7.0.3: version "7.1.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.0.tgz#36add856d746d0d99e4cc2797bba1ae2c67272fd" dependencies: @@ -2774,10 +2619,6 @@ global@^4.3.0: min-document "^2.19.0" process "~0.5.1" -globals@^9.0.0: - version "9.17.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-9.17.0.tgz#0c0ca696d9b9bb694d2e5470bd37777caad50286" - globals@^9.14.0, globals@^9.18.0: version "9.18.0" resolved "https://registry.yarnpkg.com/globals/-/globals-9.18.0.tgz#aa3896b3e69b487f17e31ed2143d69a8e30c2d8a" @@ -2872,17 +2713,7 @@ hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" -history@^4.5.1, history@^4.6.0: - version "4.6.1" - resolved "https://registry.yarnpkg.com/history/-/history-4.6.1.tgz#911cf8eb65728555a94f2b12780a0c531a14d2fd" - dependencies: - invariant "^2.2.1" - loose-envify "^1.2.0" - resolve-pathname "^2.0.0" - value-equal "^0.2.0" - warning "^3.0.0" - -history@^4.7.2: +history@^4.5.1, history@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/history/-/history-4.7.2.tgz#22b5c7f31633c5b8021c7f4a8a954ac139ee8d5b" dependencies: @@ -3100,7 +2931,7 @@ intl@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/intl/-/intl-1.2.5.tgz#82244a2190c4e419f8371f5aa34daa3420e2abde" -invariant@^2.0.0, invariant@^2.1.0, invariant@^2.1.1, invariant@^2.2.0, invariant@^2.2.1, invariant@^2.2.2: +invariant@^2.0.0, invariant@^2.1.0, invariant@^2.1.1, invariant@^2.2.1, invariant@^2.2.2: version "2.2.2" resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.2.tgz#9e1f56ac0acdb6bf303306f338be3b204ae60360" dependencies: @@ -3139,7 +2970,6 @@ is-buffer@^1.0.2: is-buffer@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.3.tgz#4ecf3fcf749cbd1e472689e109ac66261a25e725" - integrity sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw== is-builtin-module@^1.0.0: version "1.0.0" @@ -3336,7 +3166,6 @@ js-base64@^2.1.9: js-file-download@^0.4.8: version "0.4.8" resolved "https://registry.yarnpkg.com/js-file-download/-/js-file-download-0.4.8.tgz#f91c532acd685d8b16685672010b99473ca7f606" - integrity sha512-8xygX/IkjQbr/2nWqJnyc0IWOMvA1R/78HQVyexB22YZDBAEz2MG59s+ieLFKOkDFzyDDk3bezKXEjyGW5HPCw== js-tokens@^3.0.0: version "3.0.1" @@ -3346,21 +3175,13 @@ js-tokens@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b" -js-yaml@^3.13.1: +js-yaml@^3.13.1, js-yaml@^3.5.1: version "3.13.1" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.13.1.tgz#aff151b30bfdfa8e49e05da22e7415e9dfa37847" - integrity sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw== dependencies: argparse "^1.0.7" esprima "^4.0.0" -js-yaml@^3.5.1: - version "3.8.4" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.4.tgz#520b4564f86573ba96662af85a8cafa7b4b5a6f6" - dependencies: - argparse "^1.0.7" - esprima "^3.1.1" - js-yaml@~3.7.0: version "3.7.0" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.7.0.tgz#5c967ddd837a9bfdca5f2de84253abe8a1c03b80" @@ -3427,16 +3248,10 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.3.6" -jsx-ast-utils@^1.0.0: +jsx-ast-utils@^1.0.0, jsx-ast-utils@^1.3.4: version "1.4.1" resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-1.4.1.tgz#3867213e8dd79bf1e8f2300c0cfc1efb182c0df1" -jsx-ast-utils@^1.3.4: - version "1.4.0" - resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-1.4.0.tgz#5afe38868f56bc8cc7aeaef0100ba8c75bd12591" - dependencies: - object-assign "^4.1.0" - keycode@^2.1.2: version "2.1.8" resolved "https://registry.yarnpkg.com/keycode/-/keycode-2.1.8.tgz#94d2b7098215eff0e8f9a8931d5a59076c4532fb" @@ -3681,13 +3496,7 @@ minimalistic-crypto-utils@^1.0.0, minimalistic-crypto-utils@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a" -minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" - dependencies: - brace-expansion "^1.0.0" - -minimatch@^3.0.4: +minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: @@ -3715,10 +3524,6 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" -ms@0.7.2: - version "0.7.2" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" - ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -3821,16 +3626,7 @@ nopt@^4.0.1: abbrev "1" osenv "^0.1.4" -normalize-package-data@^2.3.2: - version "2.3.6" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.6.tgz#498fa420c96401f787402ba21e600def9f981fff" - dependencies: - hosted-git-info "^2.1.4" - is-builtin-module "^1.0.0" - semver "2 || 3 || 4 || 5" - validate-npm-package-license "^3.0.1" - -normalize-package-data@^2.3.4: +normalize-package-data@^2.3.2, normalize-package-data@^2.3.4: version "2.4.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" dependencies: @@ -4083,7 +3879,7 @@ path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" -path-to-regexp@^1.5.3, path-to-regexp@^1.7.0: +path-to-regexp@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.7.0.tgz#59fde0f435badacba103a84e9d3bc64e96b9937d" dependencies: @@ -4442,7 +4238,6 @@ prettier@^1.5.3: prettier@^2.0.5: version "2.6.2" resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.6.2.tgz#e26d71a18a74c3d0f0597f55f01fb6c06c206032" - integrity sha512-PkUpF+qoXTqhOeWL9fu7As8LXsIUZ1WYaJiY/a7McAQzxjk82OF0tibkFXVCDImZtWxbvojFjerkiLb0/q8mew== private@^0.1.6, private@^0.1.7: version "0.1.7" @@ -4476,7 +4271,7 @@ prop-types-extra@^1.0.1: dependencies: warning "^3.0.0" -prop-types@^15.5.10: +prop-types@^15.5.10, prop-types@^15.5.4, prop-types@^15.5.6, prop-types@^15.5.7, prop-types@^15.5.8: version "15.6.0" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.6.0.tgz#ceaf083022fc46b4a35f69e13ef75aed0d639856" dependencies: @@ -4484,13 +4279,6 @@ prop-types@^15.5.10: loose-envify "^1.3.1" object-assign "^4.1.1" -prop-types@^15.5.4, prop-types@^15.5.6, prop-types@^15.5.7, prop-types@^15.5.8: - version "15.5.10" - resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.5.10.tgz#2797dfc3126182e3a95e3dfbb2e893ddd7456154" - dependencies: - fbjs "^0.8.9" - loose-envify "^1.3.1" - protocol-buffers-schema@^2.0.2: version "2.2.0" resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-2.2.0.tgz#d29c6cd73fb655978fb6989691180db844119f61" @@ -4794,19 +4582,7 @@ react-router-redux@next: prop-types "^15.5.4" react-router "^4.1.1" -react-router@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-4.1.1.tgz#d448f3b7c1b429a6fbb03395099949c606b1fe95" - dependencies: - history "^4.6.0" - hoist-non-react-statics "^1.2.0" - invariant "^2.2.2" - loose-envify "^1.3.1" - path-to-regexp "^1.5.3" - prop-types "^15.5.4" - warning "^3.0.0" - -react-router@^4.2.0: +react-router@^4.1.1, react-router@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/react-router/-/react-router-4.2.0.tgz#61f7b3e3770daeb24062dae3eedef1b054155986" dependencies: @@ -4882,19 +4658,7 @@ read-pkg@^2.0.0: normalize-package-data "^2.3.2" path-type "^2.0.0" -"readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.1.4: - version "2.2.6" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.6.tgz#8b43aed76e71483938d12a8d46c6cf1a00b1f816" - dependencies: - buffer-shims "^1.0.0" - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "~1.0.0" - process-nextick-args "~1.0.6" - string_decoder "~0.10.x" - util-deprecate "~1.0.1" - -readable-stream@^2.0.5, readable-stream@^2.2.6, readable-stream@^2.2.9: +"readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.1.4, readable-stream@^2.2.6, readable-stream@^2.2.9: version "2.3.3" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" dependencies: @@ -4906,7 +4670,7 @@ readable-stream@^2.0.5, readable-stream@^2.2.6, readable-stream@^2.2.9: string_decoder "~1.0.3" util-deprecate "~1.0.1" -readable-stream@~2.0.0: +readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@~2.0.0: version "2.0.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e" dependencies: @@ -5011,10 +4775,6 @@ regenerate@^1.2.1: version "1.3.2" resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.3.2.tgz#d1941c67bad437e1be76433add5b385f95b19260" -regenerator-runtime@^0.10.0: - version "0.10.3" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.10.3.tgz#8c4367a904b51ea62a908ac310bf99ff90a82a3e" - regenerator-runtime@^0.10.5: version "0.10.5" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz#336c3efc1220adcedda2c9fab67b5a7955a33658" @@ -5128,10 +4888,6 @@ resolve-from@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-1.0.1.tgz#26cbfe935d1aeeeabb29bc3fe5aeb01e93d44226" -resolve-pathname@^2.0.0: - version "2.0.2" - resolved "https://registry.yarnpkg.com/resolve-pathname/-/resolve-pathname-2.0.2.tgz#e55c016eb2e9df1de98e85002282bfb38c630436" - resolve-pathname@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/resolve-pathname/-/resolve-pathname-2.2.0.tgz#7e9ae21ed815fd63ab189adeee64dc831eefa879" @@ -5161,15 +4917,15 @@ right-align@^0.1.1: dependencies: align-text "^0.1.1" -rimraf@2, rimraf@^2.2.8: - version "2.5.4" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.5.4.tgz#96800093cbf1a0c86bd95b4625467535c29dfa04" +rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" dependencies: glob "^7.0.5" -rimraf@^2.5.1, rimraf@^2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" +rimraf@^2.2.8: + version "2.5.4" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.5.4.tgz#96800093cbf1a0c86bd95b4625467535c29dfa04" dependencies: glob "^7.0.5" @@ -5528,13 +5284,7 @@ supports-color@^3.2.3: dependencies: has-flag "^1.0.0" -supports-color@^4.0.0, supports-color@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.1.0.tgz#92cc14bb3dad8928ca5656c33e19a19f20af5c7a" - dependencies: - has-flag "^2.0.0" - -supports-color@^4.2.1: +supports-color@^4.0.0, supports-color@^4.1.0, supports-color@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.2.1.tgz#65a4bb2631e90e02420dba5554c375a4754bb836" dependencies: @@ -5622,10 +5372,6 @@ to-arraybuffer@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43" -to-fast-properties@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.2.tgz#f3f5c0c3ba7299a7ef99427e44633257ade43320" - to-fast-properties@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" @@ -5803,10 +5549,6 @@ validate-npm-package-license@^3.0.1: spdx-correct "~1.0.0" spdx-expression-parse "~1.0.0" -value-equal@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/value-equal/-/value-equal-0.2.0.tgz#4f41c60a3fc011139a2ec3d3340a8998ae8b69c0" - value-equal@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/value-equal/-/value-equal-0.4.0.tgz#c5bdd2f54ee093c04839d71ce2e4758a6890abc7" From 54c11cd594cc1e9947155b03d7b06490994f8852 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 18 Apr 2022 15:04:53 +0545 Subject: [PATCH 04/69] removed print --- .vscode/settings.json | 3 +++ tasks/task_runners.py | 35 ++++------------------------------- 2 files changed, 7 insertions(+), 31 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..2b7e46d21 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.formatting.provider": "yapf" +} \ No newline at end of file diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 7175a13ce..098d1a5d4 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -339,35 +339,12 @@ def add_metadata(z,theme): h = tabular.Handler(tabular_outputs,mapping,polygon_centroid=polygon_centroid) source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir, mapping=mapping) else: - # print(mapping) - geom_json=shapely.geometry.mapping(geom) - - - # defining the api-endpoint - API_ENDPOINT = settings.GALAXY_API_URL - - - # data to be sent to api - data = {"geometry":geom_json} - - - # sending post request and saving response as response object - headers = {'Content-type': "text/plain; charset=utf-8"} - r = requests.post(url = API_ENDPOINT, data = json.dumps(data) ,headers=headers) - - # extracting response text - print(r) - response_back = r.json() - # print("Response is:%s"%response_back) - # print(job) - # h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) mapping_filter = mapping - # if job.unfiltered: - # mapping_filter = None - # print("came here ") - source = Galaxy(settings.GALAXY_API_URL,geom,use_curl=True,mapping=mapping_filter) - source.fetch() + if job.unfiltered: + mapping_filter = None + source = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping_filter) + response_back=source.fetch() LOG.debug('Source start for run: {0}'.format(run_uid)) # print("again came here") # source_path = source.path() @@ -384,14 +361,10 @@ def add_metadata(z,theme): finish_task('geopackage',[zipped]) if shp: - start_task('shp') - shp.finalize() zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) bundle_files += shp.files - print("zippedfinal") - print([zipped]) print(response_back) finish_task('shp',response_back) From 21b5e58c98bc440c2c06dec39e4f2ad8bbb21c72 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 21 Apr 2022 18:26:43 +0545 Subject: [PATCH 05/69] added shapefile support --- tasks/task_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 098d1a5d4..9fe37249c 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -344,7 +344,7 @@ def add_metadata(z,theme): if job.unfiltered: mapping_filter = None source = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping_filter) - response_back=source.fetch() + response_back=source.fetch('shp') LOG.debug('Source start for run: {0}'.format(run_uid)) # print("again came here") # source_path = source.path() From 5f36b981579ebecf3d77e62a58d4db4f66ca259a Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 2 May 2022 08:41:32 +0545 Subject: [PATCH 06/69] added geojson Format on UI --- jobs/models.py | 2 +- ui/app/components/utils.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/jobs/models.py b/jobs/models.py index f559d5b5c..292bd1776 100644 --- a/jobs/models.py +++ b/jobs/models.py @@ -83,7 +83,7 @@ def validate_export_formats(value): ) for format_name in value: - if format_name not in ['shp','geopackage','garmin_img','kml','mwm','osmand_obf','osm_pbf','osm_xml','bundle','mbtiles','full_pbf']: + if format_name not in ['shp','geojson','geopackage','garmin_img','kml','mwm','osmand_obf','osm_pbf','osm_xml','bundle','mbtiles','full_pbf']: raise ValidationError( "Bad format name: %(format_name)s", params={'format_name': format_name}, diff --git a/ui/app/components/utils.js b/ui/app/components/utils.js index 75c40b9df..d4e7c00dd 100644 --- a/ui/app/components/utils.js +++ b/ui/app/components/utils.js @@ -23,6 +23,11 @@ import { selectIsLoggedIn, selectIsLoggingIn } from "../selectors"; import styles from "../styles/utilsStyles.css"; export const AVAILABLE_EXPORT_FORMATS = { + geojson: ( + + Geojson .geojson + + ), geopackage: ( GeoPackage .gpkg @@ -77,6 +82,7 @@ export const AVAILABLE_EXPORT_FORMATS = { export const REQUIRES_FEATURE_SELECTION = { shp: true, + geojson:true, geopackage: true, garmin_img: true, kml: true, From 8f513c51fdcef93f8f9533b0242f81c783d14b67 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 2 May 2022 12:20:44 +0545 Subject: [PATCH 07/69] added geojson in task_runners --- core/settings/project.py | 2 +- requirements.txt | 1 + tasks/models.py | 20 +++++++++-- tasks/task_runners.py | 75 ++++++++++++++++++++++------------------ 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/core/settings/project.py b/core/settings/project.py index d052ee9a5..e446b34f8 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -68,7 +68,7 @@ OVERPASS_API_URL = os.getenv('OVERPASS_API_URL', 'http://overpass-api.de/api/') #url to galaxy api endpoint -GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://127.0.0.1:8080/raw-data/current-snapshot/') +GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://127.0.0.1:9090/raw-data/current-snapshot/') GENERATE_MWM = os.getenv('GENERATE_MWM','/usr/local/bin/generate_mwm.sh') GENERATOR_TOOL = os.getenv('GENERATOR_TOOL','/usr/local/bin/generator_tool') diff --git a/requirements.txt b/requirements.txt index c62e61dcd..74734e154 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,4 @@ boto3~=1.9.238 # needed for cloudwatch rasterio osm-export-tool==0.0.25 rtree +validators diff --git a/tasks/models.py b/tasks/models.py index 16ffb2fc9..ba686e30a 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -14,6 +14,8 @@ from django.contrib.gis.admin import GeoModelAdmin from django.utils.safestring import mark_safe from django.core.urlresolvers import reverse +import validators + class ExportRun(models.Model): """ @@ -87,11 +89,23 @@ def duration(self): @property def download_urls(self): def fdownload(fname): + valid=validators.url(fname) + if valid==True: + download_url = fname + filesize_bytes=self.filesize_bytes + absolute_download_url=download_url + else: + try: + filesize_bytes = os.path.getsize(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid), fname).encode('utf-8')) + except Exception: + filesize_bytes = 0 + download_url = os.path.join(settings.EXPORT_MEDIA_ROOT,str(self.run.uid), fname) + absolute_download_url=settings.HOSTNAME + download_url return { "filename":fname, - "filesize_bytes": self.filesize_bytes, - "download_url":self.filenames, - "absolute_download_url":self.filenames + "filesize_bytes": filesize_bytes, + "download_url":download_url, + "absolute_download_url":absolute_download_url } return map(fdownload, self.filenames) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 9fe37249c..49a7c8e7e 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -16,7 +16,7 @@ if not apps.ready and not settings.configured: django.setup() -import requests + import dramatiq from raven import Client @@ -32,11 +32,10 @@ import osm_export_tool.nontabular as nontabular from osm_export_tool.mapping import Mapping from osm_export_tool.geometry import load_geometry -from osm_export_tool.sources import Overpass, OsmiumTool , Galaxy +from osm_export_tool.sources import Overpass, OsmiumTool, Galaxy from osm_export_tool.package import create_package, create_posm_bundle -import json + import shapely.geometry -import shapely from .email import ( send_completion_notification, @@ -54,14 +53,10 @@ ZIP_README = """This thematic file was generated by the HOT Export Tool. For more information, visit http://export.hotosm.org . This theme includes features matching the filter: - {criteria} - clipped to the area defined by the included boundary.geojson. This theme includes the following OpenStreetMap keys: - {columns} - (c) OpenStreetMap contributors. This file is made available under the Open Database License: http://opendatacommons.org/licenses/odbl/1.0/. Any rights in individual contents of the database are licensed under the Database Contents License: http://opendatacommons.org/licenses/dbcl/1.0/ """ @@ -86,12 +81,10 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - print(" on demand") run_task_remote(run_uid) db.close_old_connections() # run_task_async_ondemand.send(run_uid) else: - run_task_async_scheduled.send(run_uid) return run @@ -106,8 +99,6 @@ def run_task_async_scheduled(run_uid): db.close_old_connections() def run_task_remote(run_uid): - print("runtaskondemand") - try: run = ExportRun.objects.get(uid=run_uid) run.status = 'RUNNING' @@ -150,20 +141,27 @@ def start_task(name): task.started_at = timezone.now() task.save() - def finish_task(name,response_back,planet_file=False): + def finish_task(name,created_files=None,response_back=None,planet_file=False): + if response_back : + print("\nResponse:") + print(response_back) LOG.debug('Task Finish: {0} for run: {1}'.format(name, run_uid)) task = ExportTask.objects.get(run__uid=run_uid, name=name) task.status = 'SUCCESS' task.finished_at = timezone.now() # assumes each file only has one part (all are zips or PBFs) - print("saved file name") - # print([basename(file.parts[0]) for file in created_files]) - task.filenames = [response_back['download_url']] + if response_back : + task.filenames = [response_back['download_url']] + else: + task.filenames = [basename(file.parts[0]) for file in created_files] if planet_file is False: - # total_bytes = 0 - # for file in created_files: - # total_bytes += file.size() - task.filesize_bytes = response_back['zip_file_size'] + if response_back: + task.filesize_bytes = int(response_back['zip_file_size'][0]) + else: + total_bytes = 0 + for file in created_files: + total_bytes += file.size() + task.filesize_bytes = total_bytes task.save() is_hdx_export = HDXExportRegion.objects.filter(job_id=run.job_id).exists() @@ -233,6 +231,7 @@ def finish_task(name,response_back,planet_file=False): if planet_file: h = tabular.Handler(tabular_outputs,mapping,polygon_centroid=polygon_centroid) source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir) + else: h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) mapping_filter = mapping @@ -318,6 +317,7 @@ def add_metadata(z,theme): geopackage = None shp = None kml = None + geojson=None tabular_outputs = [] if 'geopackage' in export_formats: @@ -329,6 +329,10 @@ def add_metadata(z,theme): shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) tabular_outputs.append(shp) start_task('shp') + + if 'geojson' in export_formats: + geojson = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping) + start_task('geojson') if 'kml' in export_formats: kml = tabular.Kml(join(stage_dir,valid_name),mapping) @@ -339,18 +343,19 @@ def add_metadata(z,theme): h = tabular.Handler(tabular_outputs,mapping,polygon_centroid=polygon_centroid) source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir, mapping=mapping) else: + if geojson is None : + h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) + mapping_filter = mapping + if job.unfiltered: + mapping_filter = None + source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) - mapping_filter = mapping - if job.unfiltered: - mapping_filter = None - source = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping_filter) - response_back=source.fetch('shp') - LOG.debug('Source start for run: {0}'.format(run_uid)) - # print("again came here") - # source_path = source.path() - LOG.debug('Source end for run: {0}'.format(run_uid)) + if geojson is None : + LOG.debug('Source start for run: {0}'.format(run_uid)) + source_path = source.path() + LOG.debug('Source end for run: {0}'.format(run_uid)) - # h.apply_file(source_path, locations=True, idx='sparse_file_array') + h.apply_file(source_path, locations=True, idx='sparse_file_array') bundle_files = [] @@ -361,12 +366,14 @@ def add_metadata(z,theme): finish_task('geopackage',[zipped]) if shp: - start_task('shp') shp.finalize() zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) bundle_files += shp.files - print(response_back) - finish_task('shp',response_back) + finish_task('shp',[zipped]) + + if geojson : + response_back=geojson.fetch('GeoJSON') + finish_task('geojson',response_back=response_back) if kml: kml.finalize() @@ -426,4 +433,4 @@ def add_metadata(z,theme): run.status = 'COMPLETED' run.finished_at = timezone.now() run.save() - LOG.debug('Finished ExportRun with id: {0}'.format(run_uid)) + LOG.debug('Finished ExportRun with id: {0}'.format(run_uid)) \ No newline at end of file From 8a529da34dd593d4032eb2b5f20460e2a55d25d4 Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma Date: Tue, 3 May 2022 14:42:57 +0545 Subject: [PATCH 08/69] added administrative boundaries --- ui/app/utils/TreeTagSettings.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ui/app/utils/TreeTagSettings.js b/ui/app/utils/TreeTagSettings.js index 7906da83e..b2c8be588 100644 --- a/ui/app/utils/TreeTagSettings.js +++ b/ui/app/utils/TreeTagSettings.js @@ -6,6 +6,12 @@ export const TAGTREE = { "Materials and Condition": {} } }, + "Administrative Boundaries":{ + "children":{ + "Boundaries":{} + } + + }, "Commercial": { "children": { "Shop": {}, @@ -181,6 +187,15 @@ export const TAGLOOKUP = { ], "where": "building IS NOT NULL" }, + "Boundaries": { + "geom_types": [ + "polygon" + ], + "keys": [ + "name" + ], + "where": "boundary='administrative'" + }, "Addresses": { "geom_types": [ "point", From f5696099c2c013c26834ad0c9900408b62c2a059 Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma Date: Tue, 3 May 2022 14:53:01 +0545 Subject: [PATCH 09/69] updated csv for administrative boundary --- ui/app/utils/tagtree.csv | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui/app/utils/tagtree.csv b/ui/app/utils/tagtree.csv index 4e40703ab..9a2377a5e 100644 --- a/ui/app/utils/tagtree.csv +++ b/ui/app/utils/tagtree.csv @@ -3,6 +3,8 @@ Buildings,Buildings,yes,,,,building IS NOT NULL,, Buildings,Building Names and Geometries,,,"point, polygon","name, building",building IS NOT NULL,,http://wiki.openstreetmap.org/wiki/Map_Features#Building Buildings,Addresses,,,"point, polygon","addr:housenumber,addr:street",building IS NOT NULL,, Buildings,Materials and Condition,,,polygon,"building:material,roof:material,access:roof",building IS NOT NULL,, +Buildings,Materials and Condition,,,polygon,"building:material,roof:material,access:roof",building IS NOT NULL,, +Administrative Boundaries,Boundaries,,,polygon,"name",boundary='administrative',, Commercial,Commercial,yes,,,,"shop IS NOT NULL OR tourism IS NOT NULL OR amenity IN ('marketplace','restaurant','fast_food','cafe','bar','pub') OR office IS NOT NULL",,http://wiki.openstreetmap.org/wiki/Map_Features#Tourism Commercial,Shop,,,"point, polygon","name, opening_hours, shop",shop IS NOT NULL,, Commercial,Supermarket,,,"point, polygon","name, opening_hours, shop",amenity='marketplace' OR shop='convenience' OR shop='supermarket',, From 39e08fbc7a7620bee6a629b551b71594d9269ae4 Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma Date: Tue, 3 May 2022 14:59:50 +0545 Subject: [PATCH 10/69] added boundary key first --- ui/app/utils/TreeTagSettings.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ui/app/utils/TreeTagSettings.js b/ui/app/utils/TreeTagSettings.js index b2c8be588..7421b5338 100644 --- a/ui/app/utils/TreeTagSettings.js +++ b/ui/app/utils/TreeTagSettings.js @@ -187,6 +187,15 @@ export const TAGLOOKUP = { ], "where": "building IS NOT NULL" }, + "Administrative Boundaries" :{ + "geom_types": [ + "polygon" + ], + "keys": [ + "name" + ], + "where": "boundary='administrative'" + }, "Boundaries": { "geom_types": [ "polygon" From c8e251cde8ff3c50f6cafd9df626aec055103954 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 3 May 2022 17:52:38 +0545 Subject: [PATCH 11/69] diverted export name logic from galaxy api response to export tool job name --- tasks/models.py | 3 ++- tasks/task_runners.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/models.py b/tasks/models.py index ba686e30a..4f4576142 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -94,6 +94,7 @@ def fdownload(fname): download_url = fname filesize_bytes=self.filesize_bytes absolute_download_url=download_url + fname=f"""{self.run.job.name}_{self.name}.zip""" else: try: filesize_bytes = os.path.getsize(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid), fname).encode('utf-8')) @@ -101,7 +102,7 @@ def fdownload(fname): filesize_bytes = 0 download_url = os.path.join(settings.EXPORT_MEDIA_ROOT,str(self.run.uid), fname) absolute_download_url=settings.HOSTNAME + download_url - return { + return { "filename":fname, "filesize_bytes": filesize_bytes, "download_url":download_url, diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 49a7c8e7e..fce7b2f44 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -156,7 +156,7 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): task.filenames = [basename(file.parts[0]) for file in created_files] if planet_file is False: if response_back: - task.filesize_bytes = int(response_back['zip_file_size'][0]) + task.filesize_bytes = int(response_back['zip_file_size'][0])*1000000 # converting to bytes since we got response in MB else: total_bytes = 0 for file in created_files: From 12997167608b78233636ef30d3832b98da461112 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 3 May 2022 18:28:28 +0545 Subject: [PATCH 12/69] binding for filename with galaxy --- tasks/task_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index fce7b2f44..55a32f8fb 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -331,7 +331,7 @@ def add_metadata(z,theme): start_task('shp') if 'geojson' in export_formats: - geojson = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping) + geojson = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) start_task('geojson') if 'kml' in export_formats: From 454752dfca75a9dc12ab0cbc60c064037ded026d Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 5 May 2022 14:01:19 +0545 Subject: [PATCH 13/69] added host and shapefile --- core/settings/project.py | 2 +- tasks/task_runners.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/core/settings/project.py b/core/settings/project.py index e446b34f8..cdd73b7ea 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -68,7 +68,7 @@ OVERPASS_API_URL = os.getenv('OVERPASS_API_URL', 'http://overpass-api.de/api/') #url to galaxy api endpoint -GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://127.0.0.1:9090/raw-data/current-snapshot/') +GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://44.203.33.53:8000/raw-data/current-snapshot/') GENERATE_MWM = os.getenv('GENERATE_MWM','/usr/local/bin/generate_mwm.sh') GENERATOR_TOOL = os.getenv('GENERATOR_TOOL','/usr/local/bin/generator_tool') diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 55a32f8fb..baa5189f2 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -90,6 +90,8 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa @dramatiq.actor(max_retries=0,queue_name='default',time_limit=1000*60*60*6) def run_task_async_ondemand(run_uid): + print("la aaye ta ni ma yeha ") + run_task_remote(run_uid) db.close_old_connections() @@ -99,6 +101,7 @@ def run_task_async_scheduled(run_uid): db.close_old_connections() def run_task_remote(run_uid): + print("run task vitra ni aaidiye lau ja") try: run = ExportRun.objects.get(uid=run_uid) run.status = 'RUNNING' @@ -326,8 +329,9 @@ def add_metadata(z,theme): start_task('geopackage') if 'shp' in export_formats: - shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) - tabular_outputs.append(shp) + shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) + # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) + # tabular_outputs.append(shp) start_task('shp') if 'geojson' in export_formats: @@ -343,14 +347,14 @@ def add_metadata(z,theme): h = tabular.Handler(tabular_outputs,mapping,polygon_centroid=polygon_centroid) source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir, mapping=mapping) else: - if geojson is None : + if geojson is None or shp is None: h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) mapping_filter = mapping if job.unfiltered: mapping_filter = None source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) - if geojson is None : + if geojson is None or shp is None: LOG.debug('Source start for run: {0}'.format(run_uid)) source_path = source.path() LOG.debug('Source end for run: {0}'.format(run_uid)) @@ -366,11 +370,9 @@ def add_metadata(z,theme): finish_task('geopackage',[zipped]) if shp: - shp.finalize() - zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) - bundle_files += shp.files - finish_task('shp',[zipped]) - + response_back=geojson.fetch('shp') + finish_task('shp',response_back=response_back) + if geojson : response_back=geojson.fetch('GeoJSON') finish_task('geojson',response_back=response_back) From ac24d4ea612dcee854fca66da4682db77380c0d6 Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma Date: Fri, 13 May 2022 18:22:55 +0545 Subject: [PATCH 14/69] changed process to run on background --- tasks/task_runners.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 49a7c8e7e..6bfe21f94 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -81,9 +81,9 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - run_task_remote(run_uid) - db.close_old_connections() - # run_task_async_ondemand.send(run_uid) + # run_task_remote(run_uid) + # db.close_old_connections() + run_task_async_ondemand.send(run_uid) else: run_task_async_scheduled.send(run_uid) return run From a64fababf4b257396090e8746ff849103469a47e Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma Date: Fri, 13 May 2022 18:27:30 +0545 Subject: [PATCH 15/69] removed comments --- tasks/task_runners.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 36154b694..0a318dd39 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -90,8 +90,7 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa @dramatiq.actor(max_retries=0,queue_name='default',time_limit=1000*60*60*6) def run_task_async_ondemand(run_uid): - print("la aaye ta ni ma yeha ") - + # print("la aaye ta ni ma yeha ") run_task_remote(run_uid) db.close_old_connections() From 0b687460a9e46f62a9cbe56d7a9758a4aa400847 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 17 May 2022 16:46:28 +0545 Subject: [PATCH 16/69] fixed pointer bug --- tasks/task_runners.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 0a318dd39..a48eb6466 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -81,16 +81,15 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - # run_task_remote(run_uid) - # db.close_old_connections() - run_task_async_ondemand.send(run_uid) + run_task_remote(run_uid) + db.close_old_connections() + # run_task_async_ondemand.send(run_uid) else: run_task_async_scheduled.send(run_uid) return run @dramatiq.actor(max_retries=0,queue_name='default',time_limit=1000*60*60*6) def run_task_async_ondemand(run_uid): - # print("la aaye ta ni ma yeha ") run_task_remote(run_uid) db.close_old_connections() @@ -100,7 +99,6 @@ def run_task_async_scheduled(run_uid): db.close_old_connections() def run_task_remote(run_uid): - print("run task vitra ni aaidiye lau ja") try: run = ExportRun.objects.get(uid=run_uid) run.status = 'RUNNING' @@ -138,6 +136,7 @@ def run_task(run_uid,run,stage_dir,download_dir): mapping = Mapping(job.feature_selection) def start_task(name): + task = ExportTask.objects.get(run__uid=run_uid, name=name) task.status = 'RUNNING' task.started_at = timezone.now() @@ -320,6 +319,7 @@ def add_metadata(z,theme): shp = None kml = None geojson=None + galaxy=False tabular_outputs = [] if 'geopackage' in export_formats: @@ -328,12 +328,13 @@ def add_metadata(z,theme): start_task('geopackage') if 'shp' in export_formats: - shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) - # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) - # tabular_outputs.append(shp) + # shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) + shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) + tabular_outputs.append(shp) start_task('shp') if 'geojson' in export_formats: + galaxy=True geojson = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) start_task('geojson') @@ -341,19 +342,17 @@ def add_metadata(z,theme): kml = tabular.Kml(join(stage_dir,valid_name),mapping) tabular_outputs.append(kml) start_task('kml') - if planet_file: h = tabular.Handler(tabular_outputs,mapping,polygon_centroid=polygon_centroid) source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir, mapping=mapping) else: - if geojson is None or shp is None: + if galaxy == False: h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) mapping_filter = mapping if job.unfiltered: mapping_filter = None source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) - - if geojson is None or shp is None: + if galaxy == False: LOG.debug('Source start for run: {0}'.format(run_uid)) source_path = source.path() LOG.debug('Source end for run: {0}'.format(run_uid)) @@ -361,6 +360,7 @@ def add_metadata(z,theme): h.apply_file(source_path, locations=True, idx='sparse_file_array') bundle_files = [] + if geopackage: geopackage.finalize() From 6cdf519d064e279c6d17254504315746f54cd8e0 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 17 May 2022 17:15:35 +0545 Subject: [PATCH 17/69] uses shapefile from galaxy --- tasks/task_runners.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index a48eb6466..03f167ba4 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -328,9 +328,10 @@ def add_metadata(z,theme): start_task('geopackage') if 'shp' in export_formats: - # shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) - shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) - tabular_outputs.append(shp) + galaxy=True + shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) + # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) + # tabular_outputs.append(shp) start_task('shp') if 'geojson' in export_formats: @@ -369,7 +370,7 @@ def add_metadata(z,theme): finish_task('geopackage',[zipped]) if shp: - response_back=geojson.fetch('shp') + response_back=shp.fetch('shp') finish_task('shp',response_back=response_back) if geojson : From 1075a6263a15ee32e9832264d1a413fc25992976 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 17 May 2022 19:05:55 +0545 Subject: [PATCH 18/69] using background task --- tasks/task_runners.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 03f167ba4..c7c509efe 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -81,9 +81,9 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - run_task_remote(run_uid) - db.close_old_connections() - # run_task_async_ondemand.send(run_uid) + # run_task_remote(run_uid) + # db.close_old_connections() + run_task_async_ondemand.send(run_uid) else: run_task_async_scheduled.send(run_uid) return run From 9b863bb3cce2a73970dc0053ceafcfdeab49acc6 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Fri, 20 May 2022 08:14:01 +0545 Subject: [PATCH 19/69] changed task_runners in default state --- tasks/task_runners.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index c7c509efe..d4b78f7aa 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -319,8 +319,6 @@ def add_metadata(z,theme): shp = None kml = None geojson=None - galaxy=False - tabular_outputs = [] if 'geopackage' in export_formats: geopackage = tabular.Geopackage(join(stage_dir,valid_name),mapping) @@ -328,14 +326,12 @@ def add_metadata(z,theme): start_task('geopackage') if 'shp' in export_formats: - galaxy=True shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) # tabular_outputs.append(shp) start_task('shp') if 'geojson' in export_formats: - galaxy=True geojson = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) start_task('geojson') @@ -347,18 +343,16 @@ def add_metadata(z,theme): h = tabular.Handler(tabular_outputs,mapping,polygon_centroid=polygon_centroid) source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir, mapping=mapping) else: - if galaxy == False: - h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) - mapping_filter = mapping - if job.unfiltered: - mapping_filter = None - source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) - if galaxy == False: - LOG.debug('Source start for run: {0}'.format(run_uid)) - source_path = source.path() - LOG.debug('Source end for run: {0}'.format(run_uid)) - - h.apply_file(source_path, locations=True, idx='sparse_file_array') + h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) + mapping_filter = mapping + if job.unfiltered: + mapping_filter = None + source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) + LOG.debug('Source start for run: {0}'.format(run_uid)) + source_path = source.path() + LOG.debug('Source end for run: {0}'.format(run_uid)) + + h.apply_file(source_path, locations=True, idx='sparse_file_array') bundle_files = [] From be36f2319eb9b032cb6bfeb00bed55f0dc7331cd Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 23 May 2022 14:20:35 +0545 Subject: [PATCH 20/69] try except block on geocode indx file reading --- api/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index d53ed1f7a..5d78fc10c 100644 --- a/api/views.py +++ b/api/views.py @@ -52,8 +52,10 @@ renderer_classes = (JSONRenderer, HOTExportApiRenderer) DIR = os.path.dirname(os.path.abspath(__file__)) -idx = index.Rtree(os.path.join(DIR,'reverse_geocode')) - +try: + idx = index.Rtree(os.path.join(DIR,'reverse_geocode')) +except: + raise ImportError("Can not read indexes") def bbox_to_geom(s): try: return GEOSGeometry(Polygon.from_bbox(s.split(',')), srid=4326) From f5ee98603d4541aa6e0891388a1ccd92cc6b642c Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 23 May 2022 14:56:24 +0545 Subject: [PATCH 21/69] debug --- api/views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/views.py b/api/views.py index 5d78fc10c..25c879b39 100644 --- a/api/views.py +++ b/api/views.py @@ -55,7 +55,8 @@ try: idx = index.Rtree(os.path.join(DIR,'reverse_geocode')) except: - raise ImportError("Can not read indexes") + pass + # raise ImportError("Can not read indexes") def bbox_to_geom(s): try: return GEOSGeometry(Polygon.from_bbox(s.split(',')), srid=4326) From c38ebef8b19c07b9f567f72e14abc603a94f3f34 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 23 May 2022 15:33:46 +0545 Subject: [PATCH 22/69] debug discontinued background task --- tasks/task_runners.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index d4b78f7aa..173548c2c 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -81,9 +81,9 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - # run_task_remote(run_uid) - # db.close_old_connections() - run_task_async_ondemand.send(run_uid) + run_task_remote(run_uid) + db.close_old_connections() + # run_task_async_ondemand.send(run_uid) else: run_task_async_scheduled.send(run_uid) return run From e5bbd83bd3ce8bd089edac5439866316a73db575 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 24 May 2022 10:29:34 +0545 Subject: [PATCH 23/69] added raise error --- api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index 25c879b39..e7bde65bb 100644 --- a/api/views.py +++ b/api/views.py @@ -55,8 +55,8 @@ try: idx = index.Rtree(os.path.join(DIR,'reverse_geocode')) except: - pass - # raise ImportError("Can not read indexes") + # pass + raise ImportError("Can not read indexes") def bbox_to_geom(s): try: return GEOSGeometry(Polygon.from_bbox(s.split(',')), srid=4326) From 318c9b1aa1af515303b2dbff462f0333ff8b2778 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 24 May 2022 17:34:29 +0545 Subject: [PATCH 24/69] activated background task back --- tasks/task_runners.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 173548c2c..d4b78f7aa 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -81,9 +81,9 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - run_task_remote(run_uid) - db.close_old_connections() - # run_task_async_ondemand.send(run_uid) + # run_task_remote(run_uid) + # db.close_old_connections() + run_task_async_ondemand.send(run_uid) else: run_task_async_scheduled.send(run_uid) return run From 6f05a9a56688fe36f54fef191a9a44e1bedf3b5d Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 24 May 2022 18:18:34 +0545 Subject: [PATCH 25/69] debug --- api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/views.py b/api/views.py index e7bde65bb..25c879b39 100644 --- a/api/views.py +++ b/api/views.py @@ -55,8 +55,8 @@ try: idx = index.Rtree(os.path.join(DIR,'reverse_geocode')) except: - # pass - raise ImportError("Can not read indexes") + pass + # raise ImportError("Can not read indexes") def bbox_to_geom(s): try: return GEOSGeometry(Polygon.from_bbox(s.split(',')), srid=4326) From 437b5269eae2d36bb5c46c1eb9c515cc6c576324 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 25 May 2022 16:43:05 +0545 Subject: [PATCH 26/69] changed basemap from mapbox to osm - removed token --- ui/app/components/MapListView.js | 4 ++-- ui/app/components/aoi/ExportAOI.js | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ui/app/components/MapListView.js b/ui/app/components/MapListView.js index cb182d189..e909ca5c4 100644 --- a/ui/app/components/MapListView.js +++ b/ui/app/components/MapListView.js @@ -154,12 +154,12 @@ export default class MapListView extends Component { source: new OSM({ attributions: [ new LayerAttribution({ - html: `© Mapbox` + html: `© OSM` }), OSM.ATTRIBUTION ], url: - "https://{a-c}.tiles.mapbox.com/styles/v1/openaerialmap/ciyx269by002w2rldex1768f5/tiles/256/{z}/{x}/{y}?access_token=pk.eyJ1Ijoib3BlbmFlcmlhbG1hcCIsImEiOiJjaXl4MjM5c20wMDBmMzNucnZtbnYwZTcxIn0.IKG5flWCS6QfpO3iOdRveg" + "http://tile.osm.org/{z}/{x}/{y}.png" }) }) ], diff --git a/ui/app/components/aoi/ExportAOI.js b/ui/app/components/aoi/ExportAOI.js index 0e0dcd844..b424cbc46 100644 --- a/ui/app/components/aoi/ExportAOI.js +++ b/ui/app/components/aoi/ExportAOI.js @@ -245,15 +245,14 @@ export class ExportAOI extends Component { // Order matters here new Tile({ source: new OSM({ - wrapX: false, attributions: [ new LayerAttribution({ - html: `© Mapbox` + html: `© OSM` }), OSM.ATTRIBUTION ], url: - "https://{a-c}.tiles.mapbox.com/styles/v1/openaerialmap/ciyx269by002w2rldex1768f5/tiles/256/{z}/{x}/{y}?access_token=pk.eyJ1Ijoib3BlbmFlcmlhbG1hcCIsImEiOiJjaXl4MjM5c20wMDBmMzNucnZtbnYwZTcxIn0.IKG5flWCS6QfpO3iOdRveg" + "http://tile.osm.org/{z}/{x}/{y}.png" }) }) ], From c3ffcc9e97904e493944735039f3b071b5f1aa9e Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 25 May 2022 17:24:12 +0545 Subject: [PATCH 27/69] added use only galaxy option on task runners --- tasks/task_runners.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index d4b78f7aa..c3c8d1f17 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -320,15 +320,22 @@ def add_metadata(z,theme): kml = None geojson=None tabular_outputs = [] + + use_only_galaxy = False + galaxy_supported_outputs = ['geojson'] + + if galaxy_supported_outputs == list(export_formats): + use_only_galaxy=True + if 'geopackage' in export_formats: geopackage = tabular.Geopackage(join(stage_dir,valid_name),mapping) tabular_outputs.append(geopackage) start_task('geopackage') if 'shp' in export_formats: - shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) - # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) - # tabular_outputs.append(shp) + # shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) + shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) + tabular_outputs.append(shp) start_task('shp') if 'geojson' in export_formats: @@ -343,16 +350,18 @@ def add_metadata(z,theme): h = tabular.Handler(tabular_outputs,mapping,polygon_centroid=polygon_centroid) source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir, mapping=mapping) else: - h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) mapping_filter = mapping if job.unfiltered: - mapping_filter = None - source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) - LOG.debug('Source start for run: {0}'.format(run_uid)) - source_path = source.path() - LOG.debug('Source end for run: {0}'.format(run_uid)) + mapping_filter = None + if use_only_galaxy == False : + h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) + source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) + if use_only_galaxy == False : + LOG.debug('Source start for run: {0}'.format(run_uid)) + source_path = source.path() + LOG.debug('Source end for run: {0}'.format(run_uid)) - h.apply_file(source_path, locations=True, idx='sparse_file_array') + h.apply_file(source_path, locations=True, idx='sparse_file_array') bundle_files = [] @@ -364,8 +373,12 @@ def add_metadata(z,theme): finish_task('geopackage',[zipped]) if shp: - response_back=shp.fetch('shp') - finish_task('shp',response_back=response_back) + shp.finalize() + zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) + bundle_files += shp.files + finish_task('shp',[zipped]) + # response_back=shp.fetch('shp') + # finish_task('shp',response_back=response_back) if geojson : response_back=geojson.fetch('GeoJSON') From 190692f96ff6d9b9989083a9526f126472a5fa1d Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 30 May 2022 11:44:36 +0545 Subject: [PATCH 28/69] sync with response from galaxy to support kb as well --- tasks/task_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index c3c8d1f17..e6a0d1b4c 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -157,7 +157,7 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): task.filenames = [basename(file.parts[0]) for file in created_files] if planet_file is False: if response_back: - task.filesize_bytes = int(response_back['zip_file_size'][0])*1000000 # converting to bytes since we got response in MB + task.filesize_bytes = int(response_back['zip_file_size_bytes'][0]) #getting filesize bytes else: total_bytes = 0 for file in created_files: From 8808ec91d2ed6c4e426d565a3e3ce01635ef9909 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 31 May 2022 18:24:51 +0545 Subject: [PATCH 29/69] added shapefile support !! --- tasks/task_runners.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index e6a0d1b4c..a2fded6b6 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -322,7 +322,7 @@ def add_metadata(z,theme): tabular_outputs = [] use_only_galaxy = False - galaxy_supported_outputs = ['geojson'] + galaxy_supported_outputs = ['geojson','shp'] if galaxy_supported_outputs == list(export_formats): use_only_galaxy=True @@ -333,9 +333,9 @@ def add_metadata(z,theme): start_task('geopackage') if 'shp' in export_formats: - # shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) - shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) - tabular_outputs.append(shp) + shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) + # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) + # tabular_outputs.append(shp) start_task('shp') if 'geojson' in export_formats: @@ -373,12 +373,12 @@ def add_metadata(z,theme): finish_task('geopackage',[zipped]) if shp: - shp.finalize() - zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) - bundle_files += shp.files - finish_task('shp',[zipped]) - # response_back=shp.fetch('shp') - # finish_task('shp',response_back=response_back) + # shp.finalize() + # zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) + # bundle_files += shp.files + # finish_task('shp',[zipped]) + response_back=shp.fetch('shp') + finish_task('shp',response_back=response_back) if geojson : response_back=geojson.fetch('GeoJSON') From 9a7d9eab9d117851cc2725456e9ac4b40e3e150e Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 31 May 2022 19:04:47 +0545 Subject: [PATCH 30/69] added run galaxy only function for if any of the option is only supplied --- tasks/task_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index a2fded6b6..90e2a2487 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -324,7 +324,7 @@ def add_metadata(z,theme): use_only_galaxy = False galaxy_supported_outputs = ['geojson','shp'] - if galaxy_supported_outputs == list(export_formats): + if galaxy_supported_outputs == list(export_formats) or set(export_formats).issubset(set(galaxy_supported_outputs)): use_only_galaxy=True if 'geopackage' in export_formats: From 61462e2ff6bc33a5a0c9bc385396df52be2474f5 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Tue, 31 May 2022 20:00:31 +0545 Subject: [PATCH 31/69] increased node points --- jobs/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/models.py b/jobs/models.py index 292bd1776..257ee5b56 100644 --- a/jobs/models.py +++ b/jobs/models.py @@ -60,7 +60,7 @@ def get_geodesic_area(geom): area = abs(int(area * 6378137 * 6378137 / 2.0 / 1000 / 1000)) return area -MAX_NODES = 10000000 +MAX_NODES = 20000000 ValidateResult = namedtuple('ValidateResult',['valid','message','params']) def check_extent(aoi,url): From b3c941f7c45a9ef7b38e84baf27ec436575845fe Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Sat, 4 Jun 2022 15:06:01 +0545 Subject: [PATCH 32/69] https://github.com/hotosm/galaxy-api/issues/237 , https://github.com/hotosm/galaxy-api/issues/236 ,https://github.com/hotosm/galaxy-api/issues/235 --- core/settings/project.py | 2 +- hdx_exports/hdx_export_set.py | 26 +++++++++++----- tasks/task_runners.py | 57 +++++++++++++++++++++-------------- 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/core/settings/project.py b/core/settings/project.py index cdd73b7ea..ec2a6eb7b 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -218,7 +218,7 @@ MATOMO_URL = os.getenv('MATOMO_URL') MATOMO_SITEID = os.getenv('MATOMO_SITEID') - +print(os.getenv('HDX_SITE', 'demo')) HDX_URL_PREFIX = Configuration.create( hdx_site=os.getenv('HDX_SITE', 'demo'), hdx_key=os.getenv('HDX_API_KEY'), diff --git a/hdx_exports/hdx_export_set.py b/hdx_exports/hdx_export_set.py index 3fac1a940..dff19b571 100644 --- a/hdx_exports/hdx_export_set.py +++ b/hdx_exports/hdx_export_set.py @@ -134,14 +134,24 @@ def datasets(self,is_private,subnational,data_update_frequency,locations,files,p resources = [] for f in files: - if 'theme' not in f.extra or f.extra['theme'] == theme.name: - file_name = os.path.basename(f.parts[0]) # only one part: the zip file - resources.append({ - 'name': file_name, - 'format': HDX_FORMATS[f.output_name], - 'description': HDX_DESCRIPTIONS[f.output_name], - 'url': os.path.join(public_dir,file_name) - }) + if isinstance(f, dict): # it is coming from galaxy + if f['theme'] == theme.name: + file_name = f['file_name'] # only one part: the zip file + resources.append({ + 'name': file_name, + 'format': HDX_FORMATS[f['output_name']], + 'description': HDX_DESCRIPTIONS[f['output_name']], + 'url': f['download_url'] + }) + else: + if 'theme' not in f.extra or f.extra['theme'] == theme.name: + file_name = os.path.basename(f.parts[0]) # only one part: the zip file + resources.append({ + 'name': file_name, + 'format': HDX_FORMATS[f.output_name], + 'description': HDX_DESCRIPTIONS[f.output_name], + 'url': os.path.join(public_dir,file_name) + }) # stable sort, but put shapefiles first for Geopreview to pick up correctly resources.sort(key=lambda x: 0 if x['format'] == 'zipped shapefile' else 1) dataset.add_update_resources(resources) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 90e2a2487..9731c8478 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -100,6 +100,7 @@ def run_task_async_scheduled(run_uid): def run_task_remote(run_uid): try: + print("inside try block") run = ExportRun.objects.get(uid=run_uid) run.status = 'RUNNING' run.started_at = timezone.now() @@ -112,6 +113,8 @@ def run_task_remote(run_uid): os.makedirs(download_dir) run_task(run_uid,run,stage_dir,download_dir) except (Job.DoesNotExist,ExportRun.DoesNotExist,ExportTask.DoesNotExist): + print("inside except block") + LOG.warn('Job was deleted - exiting.') except Exception as e: client.captureException(extra={'run_uid': run_uid}) @@ -143,21 +146,25 @@ def start_task(name): task.save() def finish_task(name,created_files=None,response_back=None,planet_file=False): - if response_back : - print("\nResponse:") - print(response_back) + # if response_back : + # print("\nResponse:") + # print(response_back) LOG.debug('Task Finish: {0} for run: {1}'.format(name, run_uid)) task = ExportTask.objects.get(run__uid=run_uid, name=name) task.status = 'SUCCESS' task.finished_at = timezone.now() # assumes each file only has one part (all are zips or PBFs) if response_back : - task.filenames = [response_back['download_url']] + # print(response_back) + task.filenames = [r['download_url'] for r in response_back] else: task.filenames = [basename(file.parts[0]) for file in created_files] if planet_file is False: if response_back: - task.filesize_bytes = int(response_back['zip_file_size_bytes'][0]) #getting filesize bytes + total_bytes = 0 + for r in response_back: + total_bytes += int(r['zip_file_size_bytes'][0]) #getting filesize bytes + task.filesize_bytes = total_bytes else: total_bytes = 0 for file in created_files: @@ -220,8 +227,9 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): start_task('geopackage') if 'shp' in export_formats: - shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) - tabular_outputs.append(shp) + shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) + # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) + # tabular_outputs.append(shp) start_task('shp') if 'kml' in export_formats: @@ -271,21 +279,26 @@ def add_metadata(z,theme): all_zips += zips if shp: - shp.finalize() - zips = [] - for file in shp.files: - # for HDX geopreview to work - # each file (_polygons, _lines) is a separate zip resource - # the zipfile must end with only .zip (not .shp.zip) - destination = join(download_dir,os.path.basename(file.parts[0]).replace('.','_') + '.zip') - with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED, True) as z: - theme = [t for t in mapping.themes if t.name == file.extra['theme']][0] - add_metadata(z,theme) - for part in file.parts: - z.write(part, os.path.basename(part)) - zips.append(osm_export_tool.File('shp',[destination],{'theme':file.extra['theme']})) - finish_task('shp',zips) - all_zips += zips + response_back=shp.fetch('shp',is_hdx_export=True) + finish_task('shp',response_back=response_back) + # shp.finalize() + # zips = [] + # for file in shp.files: + # # for HDX geopreview to work + # # each file (_polygons, _lines) is a separate zip resource + # # the zipfile must end with only .zip (not .shp.zip) + # destination = join(download_dir,os.path.basename(file.parts[0]).replace('.','_') + '.zip') + # with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED, True) as z: + # theme = [t for t in mapping.themes if t.name == file.extra['theme']][0] + # add_metadata(z,theme) + # for part in file.parts: + # z.write(part, os.path.basename(part)) + # # print(file.extra['theme']) + # zips.append(osm_export_tool.File('shp',[destination],{'theme':file.extra['theme']})) + # finish_task('shp',zips) + # all_zips += zips + + all_zips += response_back if kml: kml.finalize() From 7ddec1932ba706c69bfb8ccce46d47e658cf78bb Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Sat, 4 Jun 2022 15:47:33 +0545 Subject: [PATCH 33/69] added use only galaxy --- tasks/task_runners.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 9731c8478..2a44d52b5 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -100,7 +100,6 @@ def run_task_async_scheduled(run_uid): def run_task_remote(run_uid): try: - print("inside try block") run = ExportRun.objects.get(uid=run_uid) run.status = 'RUNNING' run.started_at = timezone.now() @@ -113,7 +112,6 @@ def run_task_remote(run_uid): os.makedirs(download_dir) run_task(run_uid,run,stage_dir,download_dir) except (Job.DoesNotExist,ExportRun.DoesNotExist,ExportTask.DoesNotExist): - print("inside except block") LOG.warn('Job was deleted - exiting.') except Exception as e: @@ -177,6 +175,8 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): planet_file = False polygon_centroid = False + galaxy_supported_outputs = ['geojson','shp'] + if is_hdx_export: planet_file = HDXExportRegion.objects.get(job_id=run.job_id).planet_file if is_partner_export: @@ -219,6 +219,10 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): geopackage = None shp = None kml = None + use_only_galaxy = False + + if galaxy_supported_outputs == list(export_formats) or set(export_formats).issubset(set(galaxy_supported_outputs)): + use_only_galaxy=True tabular_outputs = [] if 'geopackage' in export_formats: @@ -242,16 +246,19 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): source = OsmiumTool('osmium',settings.PLANET_FILE,geom,join(stage_dir,'extract.osm.pbf'),tempdir=stage_dir) else: - h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) mapping_filter = mapping if job.unfiltered: mapping_filter = None - source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) - LOG.debug('Source start for run: {0}'.format(run_uid)) - source_path = source.path() - LOG.debug('Source end for run: {0}'.format(run_uid)) - h.apply_file(source_path, locations=True, idx='sparse_file_array') + if use_only_galaxy == False : + h = tabular.Handler(tabular_outputs,mapping,clipping_geom=geom,polygon_centroid=polygon_centroid) + source = Overpass(settings.OVERPASS_API_URL,geom,join(stage_dir,'overpass.osm.pbf'),tempdir=stage_dir,use_curl=True,mapping=mapping_filter) + + if use_only_galaxy == False : + LOG.debug('Source start for run: {0}'.format(run_uid)) + source_path = source.path() + LOG.debug('Source end for run: {0}'.format(run_uid)) + h.apply_file(source_path, locations=True, idx='sparse_file_array') all_zips = [] @@ -335,7 +342,6 @@ def add_metadata(z,theme): tabular_outputs = [] use_only_galaxy = False - galaxy_supported_outputs = ['geojson','shp'] if galaxy_supported_outputs == list(export_formats) or set(export_formats).issubset(set(galaxy_supported_outputs)): use_only_galaxy=True From 19fd0a40abdcb6b09649d4721299fd84218037cd Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 21 Jul 2022 20:13:02 +0545 Subject: [PATCH 34/69] changed filename logic --- tasks/models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tasks/models.py b/tasks/models.py index 4f4576142..a6081f047 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -94,7 +94,12 @@ def fdownload(fname): download_url = fname filesize_bytes=self.filesize_bytes absolute_download_url=download_url - fname=f"""{self.run.job.name}_{self.name}.zip""" + try : + value = download_url.split('/') + name=value[-1] + fname=name + except: + fname=f"""{self.run.job.name}_{self.name}.zip""" else: try: filesize_bytes = os.path.getsize(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid), fname).encode('utf-8')) From f9bddece3ee958f46ae75b5b5dc7d3bec59e804b Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 21 Jul 2022 22:37:33 +0545 Subject: [PATCH 35/69] changed url file size for hdx --- core/settings/project.py | 2 +- tasks/models.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/settings/project.py b/core/settings/project.py index ec2a6eb7b..2ab287a11 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -68,7 +68,7 @@ OVERPASS_API_URL = os.getenv('OVERPASS_API_URL', 'http://overpass-api.de/api/') #url to galaxy api endpoint -GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://44.203.33.53:8000/raw-data/current-snapshot/') +GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://52.203.15.233:8000/raw-data/current-snapshot/') GENERATE_MWM = os.getenv('GENERATE_MWM','/usr/local/bin/generate_mwm.sh') GENERATOR_TOOL = os.getenv('GENERATOR_TOOL','/usr/local/bin/generator_tool') diff --git a/tasks/models.py b/tasks/models.py index a6081f047..1dc90578d 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -15,6 +15,7 @@ from django.utils.safestring import mark_safe from django.core.urlresolvers import reverse import validators +import requests class ExportRun(models.Model): @@ -92,7 +93,9 @@ def fdownload(fname): valid=validators.url(fname) if valid==True: download_url = fname - filesize_bytes=self.filesize_bytes + resp = requests.head("https://s3.amazonaws.com/exports-stage.hotosm.org/Hotosm_Pokhara_6de8d0bb-a75b-43d1-8b1a-160a91fc3175_f20a5cdb-11c4-43b6-adb1-5dd3b739f3d0_GeoJSON.zip") + filesize_bytes=resp.headers['Content-Length'] + # filesize_bytes=self.filesize_bytes absolute_download_url=download_url try : value = download_url.split('/') From 71fd260b43227682aa0f5b41a98948f030bf5caf Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 21 Jul 2022 22:45:04 +0545 Subject: [PATCH 36/69] added try catch in run task --- tasks/task_runners.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 2a44d52b5..c99f45ab7 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -110,7 +110,13 @@ def run_task_remote(run_uid): os.makedirs(stage_dir) if not exists(download_dir): os.makedirs(download_dir) - run_task(run_uid,run,stage_dir,download_dir) + try : + run_task(run_uid,run,stage_dir,download_dir) + except Exception as e: + LOG.warn('ExportRun {0} failed: {1}'.format(run_uid, e)) + run.status = 'FAILED' + run.finished_at = timezone.now() + run.save() except (Job.DoesNotExist,ExportRun.DoesNotExist,ExportTask.DoesNotExist): LOG.warn('Job was deleted - exiting.') From 38e2a1a6a2edecdd37b716154ff1af4ad9697ece Mon Sep 17 00:00:00 2001 From: Kshitij Raj Sharma Date: Fri, 22 Jul 2022 11:37:04 +0545 Subject: [PATCH 37/69] https://github.com/hotosm/galaxy-api/issues/273 read from download url --- tasks/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tasks/models.py b/tasks/models.py index 1dc90578d..dcd3c2977 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -93,9 +93,11 @@ def fdownload(fname): valid=validators.url(fname) if valid==True: download_url = fname - resp = requests.head("https://s3.amazonaws.com/exports-stage.hotosm.org/Hotosm_Pokhara_6de8d0bb-a75b-43d1-8b1a-160a91fc3175_f20a5cdb-11c4-43b6-adb1-5dd3b739f3d0_GeoJSON.zip") - filesize_bytes=resp.headers['Content-Length'] - # filesize_bytes=self.filesize_bytes + try: + resp = requests.head(download_url) + filesize_bytes=int(resp.headers['Content-Length']) + except: + filesize_bytes=self.filesize_bytes absolute_download_url=download_url try : value = download_url.split('/') From 22eb6b5a13ab649926d137fc7c4617e13239acbc Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Fri, 22 Jul 2022 15:39:23 +0545 Subject: [PATCH 38/69] added try except block --- tasks/task_runners.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index c99f45ab7..c67e8be51 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -292,8 +292,12 @@ def add_metadata(z,theme): all_zips += zips if shp: - response_back=shp.fetch('shp',is_hdx_export=True) - finish_task('shp',response_back=response_back) + try: + response_back=shp.fetch('shp',is_hdx_export=True) + finish_task('shp',response_back=response_back) + except Exception as ex: + raise ex + # shp.finalize() # zips = [] # for file in shp.files: @@ -402,12 +406,18 @@ def add_metadata(z,theme): # zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) # bundle_files += shp.files # finish_task('shp',[zipped]) - response_back=shp.fetch('shp') - finish_task('shp',response_back=response_back) + try : + response_back=shp.fetch('shp') + finish_task('shp',response_back=response_back) + except Exception as ex : + raise ex if geojson : - response_back=geojson.fetch('GeoJSON') - finish_task('geojson',response_back=response_back) + try: + response_back=geojson.fetch('GeoJSON') + finish_task('geojson',response_back=response_back) + except Exception as ex : + raise ex if kml: kml.finalize() From f61afd15b20fa397bd20e462d5a2cc5ffcf24e11 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 25 Jul 2022 12:13:45 +0545 Subject: [PATCH 39/69] code cleanup --- .gitignore | 1 - .vscode/settings.json | 3 - core/settings/project.py | 1 - jobs/models.py | 2 +- requirements.txt | 8 +- tasks/task_runners.py | 30 ---- ui/yarn.lock | 345 +++++++++++++++++++++++++++++++++------ 7 files changed, 298 insertions(+), 92 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index b62a4c39c..33d45f659 100755 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,5 @@ core/settings/secret.py .env node_modules ui/static/ui/js/bundle.js -ui/static/ui/js/ *.js.map secrets diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 2b7e46d21..000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.formatting.provider": "yapf" -} \ No newline at end of file diff --git a/core/settings/project.py b/core/settings/project.py index 2ab287a11..df2199592 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -218,7 +218,6 @@ MATOMO_URL = os.getenv('MATOMO_URL') MATOMO_SITEID = os.getenv('MATOMO_SITEID') -print(os.getenv('HDX_SITE', 'demo')) HDX_URL_PREFIX = Configuration.create( hdx_site=os.getenv('HDX_SITE', 'demo'), hdx_key=os.getenv('HDX_API_KEY'), diff --git a/jobs/models.py b/jobs/models.py index 257ee5b56..292bd1776 100644 --- a/jobs/models.py +++ b/jobs/models.py @@ -60,7 +60,7 @@ def get_geodesic_area(geom): area = abs(int(area * 6378137 * 6378137 / 2.0 / 1000 / 1000)) return area -MAX_NODES = 20000000 +MAX_NODES = 10000000 ValidateResult = namedtuple('ValidateResult',['valid','message','params']) def check_extent(aoi,url): diff --git a/requirements.txt b/requirements.txt index 74734e154..0f2ef5474 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ djangorestframework~=3.6.3 hdx-python-api==2.1.5 mercantile~=0.10.0 psycopg2==2.8.3 -python-openid +python-openid==2.2.5 social-auth-app-django==2.1.0 social-auth-core==1.7.0 pytz @@ -22,7 +22,7 @@ gunicorn==19.9.0 osmium~=2.15.2 dramatiq[redis,watch]~=1.7.0 boto3~=1.9.238 # needed for cloudwatch -rasterio +rasterio~=1.0.25 osm-export-tool==0.0.25 -rtree -validators +rtree==0.9.1 +validators \ No newline at end of file diff --git a/tasks/task_runners.py b/tasks/task_runners.py index c67e8be51..33be9a335 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -150,16 +150,12 @@ def start_task(name): task.save() def finish_task(name,created_files=None,response_back=None,planet_file=False): - # if response_back : - # print("\nResponse:") - # print(response_back) LOG.debug('Task Finish: {0} for run: {1}'.format(name, run_uid)) task = ExportTask.objects.get(run__uid=run_uid, name=name) task.status = 'SUCCESS' task.finished_at = timezone.now() # assumes each file only has one part (all are zips or PBFs) if response_back : - # print(response_back) task.filenames = [r['download_url'] for r in response_back] else: task.filenames = [basename(file.parts[0]) for file in created_files] @@ -238,8 +234,6 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): if 'shp' in export_formats: shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) - # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) - # tabular_outputs.append(shp) start_task('shp') if 'kml' in export_formats: @@ -297,24 +291,6 @@ def add_metadata(z,theme): finish_task('shp',response_back=response_back) except Exception as ex: raise ex - - # shp.finalize() - # zips = [] - # for file in shp.files: - # # for HDX geopreview to work - # # each file (_polygons, _lines) is a separate zip resource - # # the zipfile must end with only .zip (not .shp.zip) - # destination = join(download_dir,os.path.basename(file.parts[0]).replace('.','_') + '.zip') - # with zipfile.ZipFile(destination, 'w', zipfile.ZIP_DEFLATED, True) as z: - # theme = [t for t in mapping.themes if t.name == file.extra['theme']][0] - # add_metadata(z,theme) - # for part in file.parts: - # z.write(part, os.path.basename(part)) - # # print(file.extra['theme']) - # zips.append(osm_export_tool.File('shp',[destination],{'theme':file.extra['theme']})) - # finish_task('shp',zips) - # all_zips += zips - all_zips += response_back if kml: @@ -363,8 +339,6 @@ def add_metadata(z,theme): if 'shp' in export_formats: shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) - # shp = tabular.Shapefile(join(stage_dir,valid_name),mapping) - # tabular_outputs.append(shp) start_task('shp') if 'geojson' in export_formats: @@ -402,10 +376,6 @@ def add_metadata(z,theme): finish_task('geopackage',[zipped]) if shp: - # shp.finalize() - # zipped = create_package(join(download_dir,valid_name + '_shp.zip'),shp.files,boundary_geom=geom) - # bundle_files += shp.files - # finish_task('shp',[zipped]) try : response_back=shp.fetch('shp') finish_task('shp',response_back=response_back) diff --git a/ui/yarn.lock b/ui/yarn.lock index 95e870203..7eb52b2f7 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -1,10 +1,8 @@ # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. # yarn lockfile v1 -# yarn v0.25.2 -# node v10.13.0 -"@blueprintjs/core@^1.24.0", "@blueprintjs/core@^1.3.0": +"@blueprintjs/core@^1.24.0": version "1.30.0" resolved "https://registry.yarnpkg.com/@blueprintjs/core/-/core-1.30.0.tgz#ae1532e57757ce079cf0cc2071981cc49999011e" dependencies: @@ -17,6 +15,19 @@ tether "^1.4" tslib "^1.5.0" +"@blueprintjs/core@^1.3.0": + version "1.24.0" + resolved "https://registry.yarnpkg.com/@blueprintjs/core/-/core-1.24.0.tgz#879cf0b17d93c871c223ed478343cdda1abe45d2" + dependencies: + "@types/dom4" "^1.5.20" + "@types/tether" "^1.1.27" + classnames "^2.2" + dom4 "^1.8" + normalize.css "4.1.1" + pure-render-decorator "^1.1" + tether "^1.4" + tslib "^1.5.0" + "@blueprintjs/datetime@^1.19.0": version "1.22.0" resolved "https://registry.yarnpkg.com/@blueprintjs/datetime/-/datetime-1.22.0.tgz#a62f6234ffe50526f8bec557ee31327c9488f852" @@ -27,12 +38,6 @@ react-day-picker "^5.3.0" tslib "^1.5.0" -"@hotosm/iso-countries-languages@^1.0.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@hotosm/iso-countries-languages/-/iso-countries-languages-1.1.2.tgz#06d751c36f787dae2e172650657e9bcea3cf6f0f" - dependencies: - prettier "^2.0.5" - "@mapbox/geojson-area@^0.2.2": version "0.2.2" resolved "https://registry.yarnpkg.com/@mapbox/geojson-area/-/geojson-area-0.2.2.tgz#18d7814aa36bf23fbbcc379f8e26a22927debf10" @@ -321,11 +326,20 @@ aws4@^1.2.1: axios@^0.18.1: version "0.18.1" resolved "https://registry.yarnpkg.com/axios/-/axios-0.18.1.tgz#ff3f0de2e7b5d180e757ad98000f1081b87bcea3" + integrity sha512-0BfJq4NSfQXd+SkFdrvFbG7addhYSBA2mQwISr46pD6E5iqkWg02RAs8vyTT/j0RTnoYmeXauBuSv1qKwR179g== dependencies: follow-redirects "1.5.10" is-buffer "^2.0.2" -babel-code-frame@^6.11.0, babel-code-frame@^6.16.0, babel-code-frame@^6.22.0, babel-code-frame@^6.26.0: +babel-code-frame@^6.11.0, babel-code-frame@^6.16.0, babel-code-frame@^6.22.0: + version "6.22.0" + resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.22.0.tgz#027620bee567a88c32561574e7fd0801d33118e4" + dependencies: + chalk "^1.1.0" + esutils "^2.0.2" + js-tokens "^3.0.0" + +babel-code-frame@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-code-frame/-/babel-code-frame-6.26.0.tgz#63fd43f7dc1e3bb7ce35947db8fe369a3f58c74b" dependencies: @@ -991,14 +1005,31 @@ babel-register@^6.26.0: mkdirp "^0.5.1" source-map-support "^0.4.15" -babel-runtime@^6.11.6, babel-runtime@^6.18.0, babel-runtime@^6.2.0, babel-runtime@^6.22.0, babel-runtime@^6.26.0: +babel-runtime@^6.11.6, babel-runtime@^6.18.0, babel-runtime@^6.2.0, babel-runtime@^6.22.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.23.0.tgz#0a9489f144de70efb3ce4300accdb329e2fc543b" + dependencies: + core-js "^2.4.0" + regenerator-runtime "^0.10.0" + +babel-runtime@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-runtime/-/babel-runtime-6.26.0.tgz#965c7058668e82b55d7bfe04ff2337bc8b5647fe" dependencies: core-js "^2.4.0" regenerator-runtime "^0.11.0" -babel-template@^6.24.1, babel-template@^6.26.0, babel-template@^6.7.0: +babel-template@^6.24.1: + version "6.25.0" + resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.25.0.tgz#665241166b7c2aa4c619d71e192969552b10c071" + dependencies: + babel-runtime "^6.22.0" + babel-traverse "^6.25.0" + babel-types "^6.25.0" + babylon "^6.17.2" + lodash "^4.2.0" + +babel-template@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.26.0.tgz#de03e2d16396b069f46dd9fff8521fb1a0e35e02" dependencies: @@ -1008,7 +1039,45 @@ babel-template@^6.24.1, babel-template@^6.26.0, babel-template@^6.7.0: babylon "^6.18.0" lodash "^4.17.4" -babel-traverse@^6.23.1, babel-traverse@^6.24.1, babel-traverse@^6.26.0: +babel-template@^6.7.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.23.0.tgz#04d4f270adbb3aa704a8143ae26faa529238e638" + dependencies: + babel-runtime "^6.22.0" + babel-traverse "^6.23.0" + babel-types "^6.23.0" + babylon "^6.11.0" + lodash "^4.2.0" + +babel-traverse@^6.23.0, babel-traverse@^6.23.1: + version "6.23.1" + resolved "https://registry.yarnpkg.com/babel-traverse/-/babel-traverse-6.23.1.tgz#d3cb59010ecd06a97d81310065f966b699e14f48" + dependencies: + babel-code-frame "^6.22.0" + babel-messages "^6.23.0" + babel-runtime "^6.22.0" + babel-types "^6.23.0" + babylon "^6.15.0" + debug "^2.2.0" + globals "^9.0.0" + invariant "^2.2.0" + lodash "^4.2.0" + +babel-traverse@^6.24.1, babel-traverse@^6.25.0: + version "6.25.0" + resolved "https://registry.yarnpkg.com/babel-traverse/-/babel-traverse-6.25.0.tgz#2257497e2fcd19b89edc13c4c91381f9512496f1" + dependencies: + babel-code-frame "^6.22.0" + babel-messages "^6.23.0" + babel-runtime "^6.22.0" + babel-types "^6.25.0" + babylon "^6.17.2" + debug "^2.2.0" + globals "^9.0.0" + invariant "^2.2.0" + lodash "^4.2.0" + +babel-traverse@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-traverse/-/babel-traverse-6.26.0.tgz#46a9cbd7edcc62c8e5c064e2d2d8d0f4035766ee" dependencies: @@ -1022,7 +1091,25 @@ babel-traverse@^6.23.1, babel-traverse@^6.24.1, babel-traverse@^6.26.0: invariant "^2.2.2" lodash "^4.17.4" -babel-types@^6.19.0, babel-types@^6.23.0, babel-types@^6.24.1, babel-types@^6.26.0: +babel-types@^6.19.0, babel-types@^6.24.1, babel-types@^6.25.0: + version "6.25.0" + resolved "https://registry.yarnpkg.com/babel-types/-/babel-types-6.25.0.tgz#70afb248d5660e5d18f811d91c8303b54134a18e" + dependencies: + babel-runtime "^6.22.0" + esutils "^2.0.2" + lodash "^4.2.0" + to-fast-properties "^1.0.1" + +babel-types@^6.23.0: + version "6.23.0" + resolved "https://registry.yarnpkg.com/babel-types/-/babel-types-6.23.0.tgz#bb17179d7538bad38cd0c9e115d340f77e7e9acf" + dependencies: + babel-runtime "^6.22.0" + esutils "^2.0.2" + lodash "^4.2.0" + to-fast-properties "^1.0.1" + +babel-types@^6.26.0: version "6.26.0" resolved "https://registry.yarnpkg.com/babel-types/-/babel-types-6.26.0.tgz#a3b073f94ab49eb6fa55cd65227a334380632497" dependencies: @@ -1031,11 +1118,19 @@ babel-types@^6.19.0, babel-types@^6.23.0, babel-types@^6.24.1, babel-types@^6.26 lodash "^4.17.4" to-fast-properties "^1.0.3" -babylon@^6.17.0, babylon@^6.18.0: +babylon@^6.11.0, babylon@^6.15.0: + version "6.16.1" + resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.16.1.tgz#30c5a22f481978a9e7f8cdfdf496b11d94b404d3" + +babylon@^6.17.0, babylon@^6.17.2: + version "6.17.4" + resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.17.4.tgz#3e8b7402b88d22c3423e137a1577883b15ff869a" + +babylon@^6.18.0: version "6.18.0" resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.18.0.tgz#af2f3b88fa6f5c1e4c634d1a0f8eac4f55b395e3" -balanced-match@^0.4.2: +balanced-match@^0.4.1, balanced-match@^0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838" @@ -1099,6 +1194,14 @@ boom@2.x.x: bootstrap@~3.4.1: version "3.4.1" resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.4.1.tgz#c3a347d419e289ad11f4033e3c4132b87c081d72" + integrity sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA== + +brace-expansion@^1.0.0: + version "1.1.6" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.6.tgz#7197d7eaa9b87e648390ea61fc66c84427420df9" + dependencies: + balanced-match "^0.4.1" + concat-map "0.0.1" brace-expansion@^1.1.7: version "1.1.8" @@ -1192,6 +1295,10 @@ buffer-indexof@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/buffer-indexof/-/buffer-indexof-1.1.0.tgz#f54f647c4f4e25228baa656a2e57e43d5f270982" +buffer-shims@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-shims/-/buffer-shims-1.0.0.tgz#9978ce317388c649ad8793028c3477ef044a8b51" + buffer-xor@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" @@ -1281,7 +1388,7 @@ chain-function@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/chain-function/-/chain-function-1.0.0.tgz#0d4ab37e7e18ead0bdc47b920764118ce58733dc" -chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: +chalk@^1.0.0, chalk@^1.1.0, chalk@^1.1.1, chalk@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" dependencies: @@ -1511,7 +1618,11 @@ core-js@^1.0.0, core-js@^1.1.1: version "1.2.7" resolved "https://registry.yarnpkg.com/core-js/-/core-js-1.2.7.tgz#652294c14651db28fa93bd2d5ff2983a4f08c636" -core-js@^2.4.0, core-js@^2.5.0: +core-js@^2.4.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.4.1.tgz#4de911e667b0eae9124e34254b53aea6fc618d3e" + +core-js@^2.5.0: version "2.5.1" resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.1.tgz#ae6874dc66937789b80754ff5428df66819ca50b" @@ -1519,10 +1630,6 @@ core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" -country-iso-3-to-2@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/country-iso-3-to-2/-/country-iso-3-to-2-1.1.0.tgz#509bc7a4d2de9abfc57ad2d18fd8a40b85940a75" - create-ecdh@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/create-ecdh/-/create-ecdh-4.0.0.tgz#888c723596cdf7612f6498233eebd7a35301737d" @@ -1550,7 +1657,15 @@ create-hmac@^1.1.0, create-hmac@^1.1.2, create-hmac@^1.1.4: safe-buffer "^5.0.1" sha.js "^2.4.8" -create-react-class@^15.5.2, create-react-class@^15.6.0: +create-react-class@^15.5.2: + version "15.6.0" + resolved "https://registry.yarnpkg.com/create-react-class/-/create-react-class-15.6.0.tgz#ab448497c26566e1e29413e883207d57cfe7bed4" + dependencies: + fbjs "^0.8.9" + loose-envify "^1.3.1" + object-assign "^4.1.1" + +create-react-class@^15.6.0: version "15.6.2" resolved "https://registry.yarnpkg.com/create-react-class/-/create-react-class-15.6.2.tgz#cf1ed15f12aad7f14ef5f2dfe05e6c42f91ef02a" dependencies: @@ -1718,7 +1833,7 @@ debug@2.6.7: dependencies: ms "2.0.0" -debug@2.6.8, debug@^2.1.1, debug@^2.2.0, debug@^2.6.6, debug@^2.6.8: +debug@2.6.8, debug@^2.6.6, debug@^2.6.8: version "2.6.8" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc" dependencies: @@ -1727,9 +1842,16 @@ debug@2.6.8, debug@^2.1.1, debug@^2.2.0, debug@^2.6.6, debug@^2.6.8: debug@=3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" + integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== dependencies: ms "2.0.0" +debug@^2.1.1, debug@^2.2.0: + version "2.6.3" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.3.tgz#0f7eb8c30965ec08c72accfa0130c8b79984141d" + dependencies: + ms "0.7.2" + debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" @@ -1917,7 +2039,11 @@ ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" -electron-to-chromium@^1.2.7, electron-to-chromium@^1.3.14: +electron-to-chromium@^1.2.7: + version "1.2.8" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.2.8.tgz#22c2e6200d350da27d6050db7e3f6f85d18cf4ed" + +electron-to-chromium@^1.3.14: version "1.3.15" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.15.tgz#08397934891cbcfaebbd18b82a95b5a481138369" @@ -2187,6 +2313,10 @@ esprima@^2.6.0: version "2.7.3" resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" +esprima@^3.1.1: + version "3.1.3" + resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633" + esprima@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.0.tgz#4499eddcd1110e0b218bacf2fa7f7f59f55ca804" @@ -2346,7 +2476,19 @@ faye-websocket@~0.11.0: dependencies: websocket-driver ">=0.5.1" -fbjs@^0.8.0, fbjs@^0.8.16, fbjs@^0.8.6: +fbjs@^0.8.0: + version "0.8.14" + resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-0.8.14.tgz#d1dbe2be254c35a91e09f31f9cd50a40b2a0ed1c" + dependencies: + core-js "^1.0.0" + isomorphic-fetch "^2.1.1" + loose-envify "^1.0.0" + object-assign "^4.1.0" + promise "^7.1.1" + setimmediate "^1.0.5" + ua-parser-js "^0.7.9" + +fbjs@^0.8.16: version "0.8.16" resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-0.8.16.tgz#5e67432f550dc41b572bf55847b8aca64e5337db" dependencies: @@ -2358,7 +2500,7 @@ fbjs@^0.8.0, fbjs@^0.8.16, fbjs@^0.8.6: setimmediate "^1.0.5" ua-parser-js "^0.7.9" -fbjs@^0.8.9: +fbjs@^0.8.6, fbjs@^0.8.9: version "0.8.11" resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-0.8.11.tgz#340b590b8a2278a01ef7467c07a16da9b753db24" dependencies: @@ -2457,6 +2599,7 @@ flatten@^1.0.2: follow-redirects@1.5.10: version "1.5.10" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" + integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== dependencies: debug "=3.1.0" @@ -2590,7 +2733,7 @@ glob-parent@^2.0.0: dependencies: is-glob "^2.0.0" -glob@^7.0.0, glob@^7.0.5, glob@^7.1.2: +glob@^7.0.0, glob@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: @@ -2601,7 +2744,7 @@ glob@^7.0.0, glob@^7.0.5, glob@^7.1.2: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.3: +glob@^7.0.3, glob@^7.0.5: version "7.1.0" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.0.tgz#36add856d746d0d99e4cc2797bba1ae2c67272fd" dependencies: @@ -2619,6 +2762,10 @@ global@^4.3.0: min-document "^2.19.0" process "~0.5.1" +globals@^9.0.0: + version "9.17.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-9.17.0.tgz#0c0ca696d9b9bb694d2e5470bd37777caad50286" + globals@^9.14.0, globals@^9.18.0: version "9.18.0" resolved "https://registry.yarnpkg.com/globals/-/globals-9.18.0.tgz#aa3896b3e69b487f17e31ed2143d69a8e30c2d8a" @@ -2713,7 +2860,17 @@ hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" -history@^4.5.1, history@^4.7.2: +history@^4.5.1, history@^4.6.0: + version "4.6.1" + resolved "https://registry.yarnpkg.com/history/-/history-4.6.1.tgz#911cf8eb65728555a94f2b12780a0c531a14d2fd" + dependencies: + invariant "^2.2.1" + loose-envify "^1.2.0" + resolve-pathname "^2.0.0" + value-equal "^0.2.0" + warning "^3.0.0" + +history@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/history/-/history-4.7.2.tgz#22b5c7f31633c5b8021c7f4a8a954ac139ee8d5b" dependencies: @@ -2931,7 +3088,7 @@ intl@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/intl/-/intl-1.2.5.tgz#82244a2190c4e419f8371f5aa34daa3420e2abde" -invariant@^2.0.0, invariant@^2.1.0, invariant@^2.1.1, invariant@^2.2.1, invariant@^2.2.2: +invariant@^2.0.0, invariant@^2.1.0, invariant@^2.1.1, invariant@^2.2.0, invariant@^2.2.1, invariant@^2.2.2: version "2.2.2" resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.2.tgz#9e1f56ac0acdb6bf303306f338be3b204ae60360" dependencies: @@ -2970,6 +3127,7 @@ is-buffer@^1.0.2: is-buffer@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-2.0.3.tgz#4ecf3fcf749cbd1e472689e109ac66261a25e725" + integrity sha512-U15Q7MXTuZlrbymiz95PJpZxu8IlipAp4dtS3wOdgPXx3mqBnslrWU14kxfHB+Py/+2PVKSr37dMAgM2A4uArw== is-builtin-module@^1.0.0: version "1.0.0" @@ -3166,6 +3324,7 @@ js-base64@^2.1.9: js-file-download@^0.4.8: version "0.4.8" resolved "https://registry.yarnpkg.com/js-file-download/-/js-file-download-0.4.8.tgz#f91c532acd685d8b16685672010b99473ca7f606" + integrity sha512-8xygX/IkjQbr/2nWqJnyc0IWOMvA1R/78HQVyexB22YZDBAEz2MG59s+ieLFKOkDFzyDDk3bezKXEjyGW5HPCw== js-tokens@^3.0.0: version "3.0.1" @@ -3175,13 +3334,21 @@ js-tokens@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b" -js-yaml@^3.13.1, js-yaml@^3.5.1: +js-yaml@^3.13.1: version "3.13.1" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.13.1.tgz#aff151b30bfdfa8e49e05da22e7415e9dfa37847" + integrity sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw== dependencies: argparse "^1.0.7" esprima "^4.0.0" +js-yaml@^3.5.1: + version "3.8.4" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.4.tgz#520b4564f86573ba96662af85a8cafa7b4b5a6f6" + dependencies: + argparse "^1.0.7" + esprima "^3.1.1" + js-yaml@~3.7.0: version "3.7.0" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.7.0.tgz#5c967ddd837a9bfdca5f2de84253abe8a1c03b80" @@ -3248,10 +3415,16 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.3.6" -jsx-ast-utils@^1.0.0, jsx-ast-utils@^1.3.4: +jsx-ast-utils@^1.0.0: version "1.4.1" resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-1.4.1.tgz#3867213e8dd79bf1e8f2300c0cfc1efb182c0df1" +jsx-ast-utils@^1.3.4: + version "1.4.0" + resolved "https://registry.yarnpkg.com/jsx-ast-utils/-/jsx-ast-utils-1.4.0.tgz#5afe38868f56bc8cc7aeaef0100ba8c75bd12591" + dependencies: + object-assign "^4.1.0" + keycode@^2.1.2: version "2.1.8" resolved "https://registry.yarnpkg.com/keycode/-/keycode-2.1.8.tgz#94d2b7098215eff0e8f9a8931d5a59076c4532fb" @@ -3496,7 +3669,13 @@ minimalistic-crypto-utils@^1.0.0, minimalistic-crypto-utils@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz#f6c00c1c0b082246e5c4d99dfb8c7c083b2b582a" -minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4: +minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" + dependencies: + brace-expansion "^1.0.0" + +minimatch@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: @@ -3524,6 +3703,10 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" +ms@0.7.2: + version "0.7.2" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.2.tgz#ae25cf2512b3885a1d95d7f037868d8431124765" + ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" @@ -3626,7 +3809,16 @@ nopt@^4.0.1: abbrev "1" osenv "^0.1.4" -normalize-package-data@^2.3.2, normalize-package-data@^2.3.4: +normalize-package-data@^2.3.2: + version "2.3.6" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.3.6.tgz#498fa420c96401f787402ba21e600def9f981fff" + dependencies: + hosted-git-info "^2.1.4" + is-builtin-module "^1.0.0" + semver "2 || 3 || 4 || 5" + validate-npm-package-license "^3.0.1" + +normalize-package-data@^2.3.4: version "2.4.0" resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" dependencies: @@ -3879,7 +4071,7 @@ path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" -path-to-regexp@^1.7.0: +path-to-regexp@^1.5.3, path-to-regexp@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.7.0.tgz#59fde0f435badacba103a84e9d3bc64e96b9937d" dependencies: @@ -4235,10 +4427,6 @@ prettier@^1.5.3: version "1.7.4" resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.7.4.tgz#5e8624ae9363c80f95ec644584ecdf55d74f93fa" -prettier@^2.0.5: - version "2.6.2" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.6.2.tgz#e26d71a18a74c3d0f0597f55f01fb6c06c206032" - private@^0.1.6, private@^0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/private/-/private-0.1.7.tgz#68ce5e8a1ef0a23bb570cc28537b5332aba63ef1" @@ -4271,7 +4459,7 @@ prop-types-extra@^1.0.1: dependencies: warning "^3.0.0" -prop-types@^15.5.10, prop-types@^15.5.4, prop-types@^15.5.6, prop-types@^15.5.7, prop-types@^15.5.8: +prop-types@^15.5.10: version "15.6.0" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.6.0.tgz#ceaf083022fc46b4a35f69e13ef75aed0d639856" dependencies: @@ -4279,6 +4467,13 @@ prop-types@^15.5.10, prop-types@^15.5.4, prop-types@^15.5.6, prop-types@^15.5.7, loose-envify "^1.3.1" object-assign "^4.1.1" +prop-types@^15.5.4, prop-types@^15.5.6, prop-types@^15.5.7, prop-types@^15.5.8: + version "15.5.10" + resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.5.10.tgz#2797dfc3126182e3a95e3dfbb2e893ddd7456154" + dependencies: + fbjs "^0.8.9" + loose-envify "^1.3.1" + protocol-buffers-schema@^2.0.2: version "2.2.0" resolved "https://registry.yarnpkg.com/protocol-buffers-schema/-/protocol-buffers-schema-2.2.0.tgz#d29c6cd73fb655978fb6989691180db844119f61" @@ -4582,7 +4777,19 @@ react-router-redux@next: prop-types "^15.5.4" react-router "^4.1.1" -react-router@^4.1.1, react-router@^4.2.0: +react-router@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-4.1.1.tgz#d448f3b7c1b429a6fbb03395099949c606b1fe95" + dependencies: + history "^4.6.0" + hoist-non-react-statics "^1.2.0" + invariant "^2.2.2" + loose-envify "^1.3.1" + path-to-regexp "^1.5.3" + prop-types "^15.5.4" + warning "^3.0.0" + +react-router@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/react-router/-/react-router-4.2.0.tgz#61f7b3e3770daeb24062dae3eedef1b054155986" dependencies: @@ -4658,7 +4865,19 @@ read-pkg@^2.0.0: normalize-package-data "^2.3.2" path-type "^2.0.0" -"readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.1.4, readable-stream@^2.2.6, readable-stream@^2.2.9: +"readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.1.4: + version "2.2.6" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.6.tgz#8b43aed76e71483938d12a8d46c6cf1a00b1f816" + dependencies: + buffer-shims "^1.0.0" + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "~1.0.0" + process-nextick-args "~1.0.6" + string_decoder "~0.10.x" + util-deprecate "~1.0.1" + +readable-stream@^2.0.5, readable-stream@^2.2.6, readable-stream@^2.2.9: version "2.3.3" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c" dependencies: @@ -4670,7 +4889,7 @@ read-pkg@^2.0.0: string_decoder "~1.0.3" util-deprecate "~1.0.1" -readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@~2.0.0: +readable-stream@~2.0.0: version "2.0.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e" dependencies: @@ -4775,6 +4994,10 @@ regenerate@^1.2.1: version "1.3.2" resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.3.2.tgz#d1941c67bad437e1be76433add5b385f95b19260" +regenerator-runtime@^0.10.0: + version "0.10.3" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.10.3.tgz#8c4367a904b51ea62a908ac310bf99ff90a82a3e" + regenerator-runtime@^0.10.5: version "0.10.5" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz#336c3efc1220adcedda2c9fab67b5a7955a33658" @@ -4888,6 +5111,10 @@ resolve-from@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-1.0.1.tgz#26cbfe935d1aeeeabb29bc3fe5aeb01e93d44226" +resolve-pathname@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/resolve-pathname/-/resolve-pathname-2.0.2.tgz#e55c016eb2e9df1de98e85002282bfb38c630436" + resolve-pathname@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/resolve-pathname/-/resolve-pathname-2.2.0.tgz#7e9ae21ed815fd63ab189adeee64dc831eefa879" @@ -4917,15 +5144,15 @@ right-align@^0.1.1: dependencies: align-text "^0.1.1" -rimraf@2, rimraf@^2.5.1, rimraf@^2.6.1: - version "2.6.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" +rimraf@2, rimraf@^2.2.8: + version "2.5.4" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.5.4.tgz#96800093cbf1a0c86bd95b4625467535c29dfa04" dependencies: glob "^7.0.5" -rimraf@^2.2.8: - version "2.5.4" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.5.4.tgz#96800093cbf1a0c86bd95b4625467535c29dfa04" +rimraf@^2.5.1, rimraf@^2.6.1: + version "2.6.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.1.tgz#c2338ec643df7a1b7fe5c54fa86f57428a55f33d" dependencies: glob "^7.0.5" @@ -5284,7 +5511,13 @@ supports-color@^3.2.3: dependencies: has-flag "^1.0.0" -supports-color@^4.0.0, supports-color@^4.1.0, supports-color@^4.2.1: +supports-color@^4.0.0, supports-color@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.1.0.tgz#92cc14bb3dad8928ca5656c33e19a19f20af5c7a" + dependencies: + has-flag "^2.0.0" + +supports-color@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.2.1.tgz#65a4bb2631e90e02420dba5554c375a4754bb836" dependencies: @@ -5372,6 +5605,10 @@ to-arraybuffer@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz#7d229b1fcc637e466ca081180836a7aabff83f43" +to-fast-properties@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.2.tgz#f3f5c0c3ba7299a7ef99427e44633257ade43320" + to-fast-properties@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" @@ -5549,6 +5786,10 @@ validate-npm-package-license@^3.0.1: spdx-correct "~1.0.0" spdx-expression-parse "~1.0.0" +value-equal@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/value-equal/-/value-equal-0.2.0.tgz#4f41c60a3fc011139a2ec3d3340a8998ae8b69c0" + value-equal@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/value-equal/-/value-equal-0.4.0.tgz#c5bdd2f54ee093c04839d71ce2e4758a6890abc7" @@ -5814,4 +6055,4 @@ yargs@~3.10.0: camelcase "^1.0.2" cliui "^2.1.0" decamelize "^1.0.0" - window-size "0.1.0" + window-size "0.1.0" \ No newline at end of file From 26e183282b102c1f28c01332d51ce7096413b7bd Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 25 Jul 2022 13:44:08 +0545 Subject: [PATCH 40/69] added gejson format in hdx exports --- .gitignore | 2 +- doc/setup-development.md | 1 + tasks/task_runners.py | 47 ++++++++++++++---------- ui/app/components/HDXExportRegionForm.js | 3 +- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 33d45f659..c2fe4ae34 100755 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,6 @@ celerybeat-schedule core/settings/secret.py .env node_modules -ui/static/ui/js/bundle.js +ui/static/ui/js/* *.js.map secrets diff --git a/doc/setup-development.md b/doc/setup-development.md index f6015ba07..a8e791714 100644 --- a/doc/setup-development.md +++ b/doc/setup-development.md @@ -68,6 +68,7 @@ python manage.py migrate ```bash cd ui/ yarn install +yarn run dist # for production yarn start # will watch for changes and re-compile as necessary ``` diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 33be9a335..8176d0c53 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -177,10 +177,15 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): planet_file = False polygon_centroid = False + use_only_galaxy = False galaxy_supported_outputs = ['geojson','shp'] + if galaxy_supported_outputs == list(export_formats) or set(export_formats).issubset(set(galaxy_supported_outputs)): + use_only_galaxy=True + if is_hdx_export: - planet_file = HDXExportRegion.objects.get(job_id=run.job_id).planet_file + if use_only_galaxy is False : + planet_file = HDXExportRegion.objects.get(job_id=run.job_id).planet_file if is_partner_export: export_region = PartnerExportRegion.objects.get(job_id=run.job_id) planet_file = export_region.planet_file @@ -221,12 +226,13 @@ def finish_task(name,created_files=None,response_back=None,planet_file=False): geopackage = None shp = None kml = None - use_only_galaxy = False - - if galaxy_supported_outputs == list(export_formats) or set(export_formats).issubset(set(galaxy_supported_outputs)): - use_only_galaxy=True + geojson=None tabular_outputs = [] + if 'geojson' in export_formats: + geojson = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) + start_task('geojson') + if 'geopackage' in export_formats: geopackage = tabular.MultiGeopackage(join(stage_dir,valid_name),mapping) tabular_outputs.append(geopackage) @@ -269,6 +275,13 @@ def add_metadata(z,theme): columns = '\n'.join(columns) readme = ZIP_README.format(criteria=theme.matcher.to_sql(),columns=columns) z.writestr("README.txt", readme) + + if geojson : + try: + response_back=geojson.fetch('GeoJSON') + finish_task('geojson',response_back=response_back) + except Exception as ex : + raise ex if geopackage: geopackage.finalize() @@ -326,11 +339,10 @@ def add_metadata(z,theme): kml = None geojson=None tabular_outputs = [] - - use_only_galaxy = False - if galaxy_supported_outputs == list(export_formats) or set(export_formats).issubset(set(galaxy_supported_outputs)): - use_only_galaxy=True + if 'geojson' in export_formats: + geojson = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) + start_task('geojson') if 'geopackage' in export_formats: geopackage = tabular.Geopackage(join(stage_dir,valid_name),mapping) @@ -341,10 +353,6 @@ def add_metadata(z,theme): shp = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) start_task('shp') - if 'geojson' in export_formats: - geojson = Galaxy(settings.GALAXY_API_URL,geom,mapping=mapping,file_name=valid_name) - start_task('geojson') - if 'kml' in export_formats: kml = tabular.Kml(join(stage_dir,valid_name),mapping) tabular_outputs.append(kml) @@ -368,6 +376,12 @@ def add_metadata(z,theme): bundle_files = [] + if geojson : + try: + response_back=geojson.fetch('GeoJSON') + finish_task('geojson',response_back=response_back) + except Exception as ex : + raise ex if geopackage: geopackage.finalize() @@ -382,13 +396,6 @@ def add_metadata(z,theme): except Exception as ex : raise ex - if geojson : - try: - response_back=geojson.fetch('GeoJSON') - finish_task('geojson',response_back=response_back) - except Exception as ex : - raise ex - if kml: kml.finalize() zipped = create_package(join(download_dir,valid_name + '_kml.zip'),kml.files,boundary_geom=geom) diff --git a/ui/app/components/HDXExportRegionForm.js b/ui/app/components/HDXExportRegionForm.js index ffd29cf40..ea54950d1 100644 --- a/ui/app/components/HDXExportRegionForm.js +++ b/ui/app/components/HDXExportRegionForm.js @@ -55,6 +55,7 @@ import { const FORM_NAME = "HDXExportRegionForm"; const HDX_EXPORT_FORMATS = { + geojson:AVAILABLE_EXPORT_FORMATS.geojson, shp: AVAILABLE_EXPORT_FORMATS.shp, geopackage: AVAILABLE_EXPORT_FORMATS.geopackage, garmin_img: AVAILABLE_EXPORT_FORMATS.garmin_img, @@ -957,7 +958,7 @@ Railways: schedule_period: "daily", schedule_hour: 0, subnational: true, - export_formats: ["shp", "geopackage", "kml", "garmin_img"], + export_formats: ["geojson","shp"], buffer_aoi: false, planet_file: false }, From 9692830f9d6e72f9f864f4b08fec8ba46b3d347f Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 25 Jul 2022 15:07:56 +0545 Subject: [PATCH 41/69] upadted hdx dataset format --- hdx_exports/hdx_export_set.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hdx_exports/hdx_export_set.py b/hdx_exports/hdx_export_set.py index dff19b571..242d9bb16 100644 --- a/hdx_exports/hdx_export_set.py +++ b/hdx_exports/hdx_export_set.py @@ -87,6 +87,7 @@ def hdx_note(self, theme): def datasets(self,is_private,subnational,data_update_frequency,locations,files,public_dir): HDX_FORMATS = { + 'geojson':'GeoJSON', 'shp': 'SHP', 'geopackage': 'Geopackage', 'garmin_img': 'Garmin IMG', @@ -94,6 +95,7 @@ def datasets(self,is_private,subnational,data_update_frequency,locations,files,p } HDX_DESCRIPTIONS = { + 'geojson': 'Geographic JavaScript Object Notation', 'shp':'ESRI Shapefile', 'geopackage':'Geopackage, SQLite compatible', 'garmin_img':'.IMG for Garmin GPS Devices (All OSM layers for area)', From 6183f74b8a01d5d02c3c04d97471679b3824dba5 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 25 Jul 2022 15:22:19 +0545 Subject: [PATCH 42/69] fixed bug on hdx exports --- tasks/task_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 8176d0c53..220eacb76 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -278,7 +278,7 @@ def add_metadata(z,theme): if geojson : try: - response_back=geojson.fetch('GeoJSON') + response_back=geojson.fetch('GeoJSON',is_hdx_export=True) finish_task('geojson',response_back=response_back) except Exception as ex : raise ex From df1af29afa4e8de70c7338dec00796d77ff6d217 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 25 Jul 2022 15:43:58 +0545 Subject: [PATCH 43/69] added response to zip --- tasks/task_runners.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 220eacb76..0a09c83ce 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -282,7 +282,7 @@ def add_metadata(z,theme): finish_task('geojson',response_back=response_back) except Exception as ex : raise ex - + all_zips += response_back if geopackage: geopackage.finalize() zips = [] From 061dbbc9f5394b58e0a5213bcdb50180b029867f Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 25 Jul 2022 17:46:07 +0545 Subject: [PATCH 44/69] added geojson working --- hdx_exports/hdx_export_set.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hdx_exports/hdx_export_set.py b/hdx_exports/hdx_export_set.py index 242d9bb16..2b032da01 100644 --- a/hdx_exports/hdx_export_set.py +++ b/hdx_exports/hdx_export_set.py @@ -87,7 +87,7 @@ def hdx_note(self, theme): def datasets(self,is_private,subnational,data_update_frequency,locations,files,public_dir): HDX_FORMATS = { - 'geojson':'GeoJSON', + 'geojson':'geojson', 'shp': 'SHP', 'geopackage': 'Geopackage', 'garmin_img': 'Garmin IMG', @@ -95,7 +95,7 @@ def datasets(self,is_private,subnational,data_update_frequency,locations,files,p } HDX_DESCRIPTIONS = { - 'geojson': 'Geographic JavaScript Object Notation', + 'geojson': 'geojson', 'shp':'ESRI Shapefile', 'geopackage':'Geopackage, SQLite compatible', 'garmin_img':'.IMG for Garmin GPS Devices (All OSM layers for area)', From 2e9ebbf39be968bcca0eac8a301deddb5bb7e7fa Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 25 Jul 2022 19:50:39 +0545 Subject: [PATCH 45/69] fixed key error on geojson on hdx exports --- hdx_exports/hdx_export_set.py | 4 ++-- tasks/task_runners.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hdx_exports/hdx_export_set.py b/hdx_exports/hdx_export_set.py index 2b032da01..5ba8931f3 100644 --- a/hdx_exports/hdx_export_set.py +++ b/hdx_exports/hdx_export_set.py @@ -87,7 +87,7 @@ def hdx_note(self, theme): def datasets(self,is_private,subnational,data_update_frequency,locations,files,public_dir): HDX_FORMATS = { - 'geojson':'geojson', + 'GeoJSON':'geojson', 'shp': 'SHP', 'geopackage': 'Geopackage', 'garmin_img': 'Garmin IMG', @@ -95,7 +95,7 @@ def datasets(self,is_private,subnational,data_update_frequency,locations,files,p } HDX_DESCRIPTIONS = { - 'geojson': 'geojson', + 'GeoJSON': 'geojson', 'shp':'ESRI Shapefile', 'geopackage':'Geopackage, SQLite compatible', 'garmin_img':'.IMG for Garmin GPS Devices (All OSM layers for area)', diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 0a09c83ce..01a45fdaf 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -85,6 +85,8 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa # db.close_old_connections() run_task_async_ondemand.send(run_uid) else: + # run_task_remote(run_uid) + # db.close_old_connections() run_task_async_scheduled.send(run_uid) return run @@ -113,6 +115,8 @@ def run_task_remote(run_uid): try : run_task(run_uid,run,stage_dir,download_dir) except Exception as e: + client.captureException(extra={'run_uid': run_uid}) + LOG.warn(traceback.format_exc()) LOG.warn('ExportRun {0} failed: {1}'.format(run_uid, e)) run.status = 'FAILED' run.finished_at = timezone.now() From f28f75a59d01e38d538d327a4a083fe34a98aaf8 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Mon, 25 Jul 2022 19:58:24 +0545 Subject: [PATCH 46/69] added geojson as default #410 --- ui/app/components/ExportForm.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/app/components/ExportForm.js b/ui/app/components/ExportForm.js index b39f1a121..f14ec1936 100644 --- a/ui/app/components/ExportForm.js +++ b/ui/app/components/ExportForm.js @@ -329,7 +329,7 @@ buildings: - building where: building IS NOT NULL `.trim(), - export_formats: ["geopackage"] + export_formats: ["geojson"] } }; }; From 937ddc8f03674748d3f78371c80ec7c4b11ebdb8 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Thu, 28 Jul 2022 11:03:15 +0545 Subject: [PATCH 47/69] changed default source for galaxyapi --- core/settings/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/settings/project.py b/core/settings/project.py index df2199592..7a007bef3 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -68,7 +68,7 @@ OVERPASS_API_URL = os.getenv('OVERPASS_API_URL', 'http://overpass-api.de/api/') #url to galaxy api endpoint -GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://52.203.15.233:8000/raw-data/current-snapshot/') +GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://52.203.15.233:8000/v1/raw-data/current-snapshot/') GENERATE_MWM = os.getenv('GENERATE_MWM','/usr/local/bin/generate_mwm.sh') GENERATOR_TOOL = os.getenv('GENERATOR_TOOL','/usr/local/bin/generator_tool') From 61bf42d37b62b5514b78ae0a447136d1a4a34e9e Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Fri, 29 Jul 2022 17:44:14 +0545 Subject: [PATCH 48/69] added logic for user friendly download name --- tasks/models.py | 16 ++++++++++------ tasks/task_runners.py | 41 +++++++++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/tasks/models.py b/tasks/models.py index dcd3c2977..3d4900870 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -93,18 +93,22 @@ def fdownload(fname): valid=validators.url(fname) if valid==True: download_url = fname - try: - resp = requests.head(download_url) - filesize_bytes=int(resp.headers['Content-Length']) - except: - filesize_bytes=self.filesize_bytes absolute_download_url=download_url try : value = download_url.split('/') name=value[-1] - fname=name + split_name=name.split('_') + download_name=f"{split_name[0]}_{split_name[-1]}" # getting human redable name ignoring unique id + fname=download_name except: fname=f"""{self.run.job.name}_{self.name}.zip""" + try: + with open(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid),f"{name}_size.txt")) as f : + size=f.readline() + filesize_bytes=int(size) + except: + filesize_bytes=0 + else: try: filesize_bytes = os.path.getsize(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid), fname).encode('utf-8')) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 01a45fdaf..ab42caa77 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -81,13 +81,13 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - # run_task_remote(run_uid) - # db.close_old_connections() - run_task_async_ondemand.send(run_uid) + run_task_remote(run_uid) + db.close_old_connections() + # run_task_async_ondemand.send(run_uid) else: - # run_task_remote(run_uid) - # db.close_old_connections() - run_task_async_scheduled.send(run_uid) + run_task_remote(run_uid) + db.close_old_connections() + # run_task_async_scheduled.send(run_uid) return run @dramatiq.actor(max_retries=0,queue_name='default',time_limit=1000*60*60*6) @@ -283,6 +283,13 @@ def add_metadata(z,theme): if geojson : try: response_back=geojson.fetch('GeoJSON',is_hdx_export=True) + try: + for r in response_back: + size_path=join(download_dir,f"{r['download_url'].split('/')[-1]}_size.txt") + with open(size_path, 'w') as f: + f.write(str(r['zip_file_size_bytes'][0])) + except: + LOG.error("Can not write filesize to text") finish_task('geojson',response_back=response_back) except Exception as ex : raise ex @@ -305,6 +312,13 @@ def add_metadata(z,theme): if shp: try: response_back=shp.fetch('shp',is_hdx_export=True) + try: + for r in response_back: + size_path=join(download_dir,f"{r['download_url'].split('/')[-1]}_size.txt") + with open(size_path, 'w') as f: + f.write(str(r['zip_file_size_bytes'][0])) + except: + LOG.error("Can not write filesize to text") finish_task('shp',response_back=response_back) except Exception as ex: raise ex @@ -383,7 +397,15 @@ def add_metadata(z,theme): if geojson : try: response_back=geojson.fetch('GeoJSON') + try: + for r in response_back: + size_path=join(download_dir,f"{r['download_url'].split('/')[-1]}_size.txt") + with open(size_path, 'w') as f: + f.write(str(r['zip_file_size_bytes'][0])) + except: + LOG.error("Can not write filesize to text") finish_task('geojson',response_back=response_back) + except Exception as ex : raise ex @@ -396,6 +418,13 @@ def add_metadata(z,theme): if shp: try : response_back=shp.fetch('shp') + try: + for r in response_back: + size_path=join(download_dir,f"{r['download_url'].split('/')[-1]}_size.txt") + with open(size_path, 'w') as f: + f.write(str(r['zip_file_size_bytes'][0])) + except: + LOG.error("Can not write filesize to text") finish_task('shp',response_back=response_back) except Exception as ex : raise ex From d35557ad988547041ce970445bc04b6cf92feb8b Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Fri, 29 Jul 2022 19:05:41 +0545 Subject: [PATCH 49/69] reverted local setup change --- tasks/task_runners.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index ab42caa77..f639bc071 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -81,13 +81,13 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - run_task_remote(run_uid) - db.close_old_connections() - # run_task_async_ondemand.send(run_uid) + # run_task_remote(run_uid) + # db.close_old_connections() + run_task_async_ondemand.send(run_uid) else: - run_task_remote(run_uid) - db.close_old_connections() - # run_task_async_scheduled.send(run_uid) + # run_task_remote(run_uid) + # db.close_old_connections() + run_task_async_scheduled.send(run_uid) return run @dramatiq.actor(max_retries=0,queue_name='default',time_limit=1000*60*60*6) From 5abc7cca3bec2a15bb6d4eb6c6f93b5ce752015f Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 10:24:22 +0545 Subject: [PATCH 50/69] changed galaxy api url to production url as default --- core/settings/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/settings/project.py b/core/settings/project.py index 7a007bef3..df6fc82e1 100644 --- a/core/settings/project.py +++ b/core/settings/project.py @@ -68,7 +68,7 @@ OVERPASS_API_URL = os.getenv('OVERPASS_API_URL', 'http://overpass-api.de/api/') #url to galaxy api endpoint -GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'http://52.203.15.233:8000/v1/raw-data/current-snapshot/') +GALAXY_API_URL = os.getenv('GALAXY_API_URL', 'https://galaxy-api.hotosm.org/v1/raw-data/current-snapshot/') GENERATE_MWM = os.getenv('GENERATE_MWM','/usr/local/bin/generate_mwm.sh') GENERATOR_TOOL = os.getenv('GENERATOR_TOOL','/usr/local/bin/generator_tool') From 634d3f7546fd98f4a1f5aeb7da42da9f40264cd0 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 10:35:37 +0545 Subject: [PATCH 51/69] removed hdxtests on makefile --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7f909f07e..050b2a62b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ django_test: python manage.py test api.tests jobs.tests tasks.tests -test: - nosetests hdx_exports/tests/test_hdx_export_set.py + worker: PYTHONPATH=../osm-export-tool-python/ DJANGO_SETTINGS_MODULE=core.settings.project dramatiq tasks.task_runners --processes 1 --threads 1 \ No newline at end of file From a350dd957d5e714a9a26a686c8f8d9799fdce617 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 10:38:59 +0545 Subject: [PATCH 52/69] brought tests back --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 050b2a62b..7f909f07e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ django_test: python manage.py test api.tests jobs.tests tasks.tests - +test: + nosetests hdx_exports/tests/test_hdx_export_set.py worker: PYTHONPATH=../osm-export-tool-python/ DJANGO_SETTINGS_MODULE=core.settings.project dramatiq tasks.task_runners --processes 1 --threads 1 \ No newline at end of file From d42113cb4958c9792ca8ee928d355786eb3c40ef Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 10:53:13 +0545 Subject: [PATCH 53/69] updated circle ci to python 3.10 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9af344155..d3da4cb2b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: circleci/python:3.6.8-stretch + - image: cimg/python:3.10.5 steps: - checkout - run: From b9637f7a76a25f93cddc31cf566804c2f4bb1e4e Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 10:56:49 +0545 Subject: [PATCH 54/69] changed python to 3.8 --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d3da4cb2b..52393b0d6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: cimg/python:3.10.5 + - image: circleci/python:3.8.0-stretch steps: - checkout - run: From df21e3ee18d2b7299797b2dbe72899d324455971 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 10:58:48 +0545 Subject: [PATCH 55/69] changed to python default --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 52393b0d6..58ac9d9e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: circleci/python:3.8.0-stretch + - image: circleci/python steps: - checkout - run: From a521c4dce6f6d7eb15da0632a93ed2dc56297dd6 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:00:20 +0545 Subject: [PATCH 56/69] added get update --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 58ac9d9e5..f7890b716 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,6 +9,7 @@ jobs: name: Run tests command: | set -x + sudo apt-get update sudo apt-get install libgdal-dev osmctools gdal-bin libspatialindex-dev virtualenv ./venv source venv/bin/activate From 7569deeae2bac9c4361ac0cfda0f79c9937d0873 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:12:24 +0545 Subject: [PATCH 57/69] added github action --- .circleci/config.yml | 3 +-- .github/config.yml | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 .github/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index f7890b716..9af344155 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,14 +2,13 @@ version: 2 jobs: build: docker: - - image: circleci/python + - image: circleci/python:3.6.8-stretch steps: - checkout - run: name: Run tests command: | set -x - sudo apt-get update sudo apt-get install libgdal-dev osmctools gdal-bin libspatialindex-dev virtualenv ./venv source venv/bin/activate diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 000000000..7a2f6cf11 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,37 @@ +name: Unit Test +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + deploy: + runs-on: + ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install gdal + run: | + sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable + sudo apt-get update + sudo apt-get install libgdal-dev osmctools gdal-bin libspatialindex-dev + + - name: Install Dependencies + run: | + + python -m pip install --upgrade pip + virtualenv ./venv + export CPLUS_INCLUDE_PATH=/usr/include/gdal + export C_INCLUDE_PATH=/usr/include/gdal + pip install -r requirements.txt + - name: Run Tests + run: | + make test \ No newline at end of file From 3224c1c54d4125bc85ac37ff0af61e6859e0d7a0 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:13:34 +0545 Subject: [PATCH 58/69] run github action on push feature/galaxy --- .github/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/config.yml b/.github/config.yml index 7a2f6cf11..1c9c549fd 100644 --- a/.github/config.yml +++ b/.github/config.yml @@ -3,6 +3,7 @@ on: push: branches: - master + - feature/galaxy pull_request: branches: - master From a9b89398e3f5bc235023df91f8e2b286831e4296 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:14:45 +0545 Subject: [PATCH 59/69] moved to folder workflow --- .github/{ => workflows}/config.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{ => workflows}/config.yml (100%) diff --git a/.github/config.yml b/.github/workflows/config.yml similarity index 100% rename from .github/config.yml rename to .github/workflows/config.yml From 75da08cae41082f0b21ef7a9ae263571f5a5fac5 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:15:33 +0545 Subject: [PATCH 60/69] reverted pr link in yml --- .github/workflows/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 1c9c549fd..7a2f6cf11 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -3,7 +3,6 @@ on: push: branches: - master - - feature/galaxy pull_request: branches: - master From aaee6064b4543017dfa72e1fb9f8bd07bbf24669 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:18:18 +0545 Subject: [PATCH 61/69] removed virtual env part --- .github/workflows/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 7a2f6cf11..ffd3f9ea4 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -28,7 +28,6 @@ jobs: run: | python -m pip install --upgrade pip - virtualenv ./venv export CPLUS_INCLUDE_PATH=/usr/include/gdal export C_INCLUDE_PATH=/usr/include/gdal pip install -r requirements.txt From 9cc66eab0471650f4026dcb22136fb9b7b54eccc Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:24:58 +0545 Subject: [PATCH 62/69] added nose --- .github/workflows/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index ffd3f9ea4..f3022f1ec 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -30,6 +30,7 @@ jobs: python -m pip install --upgrade pip export CPLUS_INCLUDE_PATH=/usr/include/gdal export C_INCLUDE_PATH=/usr/include/gdal + pip install nose pip install -r requirements.txt - name: Run Tests run: | From c67f44fed9cfd64902fd7144cdf0874eb79a130f Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:38:58 +0545 Subject: [PATCH 63/69] changing version of nose --- .github/workflows/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index f3022f1ec..5ad213ae5 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -30,7 +30,7 @@ jobs: python -m pip install --upgrade pip export CPLUS_INCLUDE_PATH=/usr/include/gdal export C_INCLUDE_PATH=/usr/include/gdal - pip install nose + pip install nose==1.3.7 pip install -r requirements.txt - name: Run Tests run: | From a8a2f078d7fee87835b1c1486dcc81138fb2d935 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:43:19 +0545 Subject: [PATCH 64/69] added requirements dev --- .github/workflows/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 5ad213ae5..2d68a26d4 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -26,12 +26,9 @@ jobs: - name: Install Dependencies run: | - - python -m pip install --upgrade pip export CPLUS_INCLUDE_PATH=/usr/include/gdal export C_INCLUDE_PATH=/usr/include/gdal - pip install nose==1.3.7 - pip install -r requirements.txt + pip install -r requirements-dev.txt - name: Run Tests run: | make test \ No newline at end of file From 58c135b60b60fce62547a2326881b79fe96a72cd Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:51:45 +0545 Subject: [PATCH 65/69] changed python to 3.8.10 --- .github/workflows/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 2d68a26d4..ab1a865be 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python 3.8 uses: actions/setup-python@v1 with: - python-version: 3.8 + python-version: 3.8.10 - name: Install gdal run: | From 4f2fbe0c5bf98e57986949930c0c4d8a33716ca3 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:53:18 +0545 Subject: [PATCH 66/69] changed python to 3.9 --- .github/workflows/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index ab1a865be..3e89f11fa 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -13,10 +13,10 @@ jobs: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Set up Python 3.8 + - name: Set up Python 3 uses: actions/setup-python@v1 with: - python-version: 3.8.10 + python-version: 3.9.13 - name: Install gdal run: | From d16f9549f22dd95bf5ebd1b5a38a14615ee88cd5 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 11:56:29 +0545 Subject: [PATCH 67/69] changed to 3.8.13 --- .github/workflows/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/config.yml b/.github/workflows/config.yml index 3e89f11fa..dcfed381b 100644 --- a/.github/workflows/config.yml +++ b/.github/workflows/config.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python 3 uses: actions/setup-python@v1 with: - python-version: 3.9.13 + python-version: 3.8.13 - name: Install gdal run: | From d222108f2b0dfc6b16128abb8f57ac5ef258c006 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 12:01:24 +0545 Subject: [PATCH 68/69] check if rasterio is cause --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0f2ef5474..184c2484e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ gunicorn==19.9.0 osmium~=2.15.2 dramatiq[redis,watch]~=1.7.0 boto3~=1.9.238 # needed for cloudwatch -rasterio~=1.0.25 +# rasterio~=1.0.25 osm-export-tool==0.0.25 rtree==0.9.1 validators \ No newline at end of file From 7b46d38a19352885610f1ee3ecb930534105d523 Mon Sep 17 00:00:00 2001 From: itskshitiz321 Date: Wed, 3 Aug 2022 12:20:14 +0545 Subject: [PATCH 69/69] brought back rasterio --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 184c2484e..0f2ef5474 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ gunicorn==19.9.0 osmium~=2.15.2 dramatiq[redis,watch]~=1.7.0 boto3~=1.9.238 # needed for cloudwatch -# rasterio~=1.0.25 +rasterio~=1.0.25 osm-export-tool==0.0.25 rtree==0.9.1 validators \ No newline at end of file