Use updated uri for gcs batch reqs (#2998)

* Use correct batch uri for GCS client requests * bump google-api-python-client version lower bound Co-authored-by: Jack Dingilian <[email protected]>
spotify · Sep 18, 2020 · 3556ec2 · 3556ec2
1 parent 7d16f29
commit 3556ec2
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/luigi/contrib/gcs.py b/luigi/contrib/gcs.py
@@ -60,6 +60,9 @@
 # Maximum number of sleeps for eventual consistency.
 EVENTUAL_CONSISTENCY_MAX_SLEEPS = 300
 
+# Uri for batch requests
+GCS_BATCH_URI = 'https://storage.googleapis.com/batch/storage/v1'
+
 
 def _wait_for_consistency(checker):
     """Eventual consistency: wait until GCS reports something is true.
@@ -231,7 +234,7 @@ def remove(self, path, recursive=True):
                 raise InvalidDeleteException(
                     'Path {} is a directory. Must use recursive delete'.format(path))
 
-            req = http.BatchHttpRequest()
+            req = http.BatchHttpRequest(batch_uri=GCS_BATCH_URI)
             for it in self._list_iter(bucket, self._add_path_delimiter(obj)):
                 req.add(self.client.objects().delete(bucket=bucket, object=it['name']))
             req.execute()

diff --git a/tox.ini b/tox.ini
@@ -23,7 +23,7 @@ deps =
     cdh,hdp: hdfs>=2.0.4,<3.0.0
     postgres: psycopg2<3.0
     mysql-connector-python>=8.0.12
-    gcloud: google-api-python-client>=1.4.0,<2.0
+    gcloud: google-api-python-client>=1.6.6,<2.0
     avro-python3
     gcloud: google-auth==1.4.1
     gcloud: google-auth-httplib2==0.0.3