From 3556ec275b1bf44537966c757306cf4c89c0e52f Mon Sep 17 00:00:00 2001 From: Jack Dingilian Date: Fri, 18 Sep 2020 14:35:48 -0400 Subject: [PATCH] Use updated uri for gcs batch reqs (#2998) * Use correct batch uri for GCS client requests * bump google-api-python-client version lower bound Co-authored-by: Jack Dingilian --- luigi/contrib/gcs.py | 5 ++++- tox.ini | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/luigi/contrib/gcs.py b/luigi/contrib/gcs.py index a5209d5f9a..3a7d5f67a6 100644 --- a/luigi/contrib/gcs.py +++ b/luigi/contrib/gcs.py @@ -60,6 +60,9 @@ # Maximum number of sleeps for eventual consistency. EVENTUAL_CONSISTENCY_MAX_SLEEPS = 300 +# Uri for batch requests +GCS_BATCH_URI = 'https://storage.googleapis.com/batch/storage/v1' + def _wait_for_consistency(checker): """Eventual consistency: wait until GCS reports something is true. @@ -231,7 +234,7 @@ def remove(self, path, recursive=True): raise InvalidDeleteException( 'Path {} is a directory. Must use recursive delete'.format(path)) - req = http.BatchHttpRequest() + req = http.BatchHttpRequest(batch_uri=GCS_BATCH_URI) for it in self._list_iter(bucket, self._add_path_delimiter(obj)): req.add(self.client.objects().delete(bucket=bucket, object=it['name'])) req.execute() diff --git a/tox.ini b/tox.ini index 4753d2e50a..362838dfd4 100644 --- a/tox.ini +++ b/tox.ini @@ -23,7 +23,7 @@ deps = cdh,hdp: hdfs>=2.0.4,<3.0.0 postgres: psycopg2<3.0 mysql-connector-python>=8.0.12 - gcloud: google-api-python-client>=1.4.0,<2.0 + gcloud: google-api-python-client>=1.6.6,<2.0 avro-python3 gcloud: google-auth==1.4.1 gcloud: google-auth-httplib2==0.0.3