Delete all files with their previous versions

onadata/apps/logger/management/commands/remove_s3_orphans.py

@@ -73,8 +73,8 @@ def handle(self, *args, **kwargs):
 
         Bucket._get_all = _get_all
 
-        s3 = get_storage_class('storages.backends.s3boto.S3BotoStorage')()
-        all_files = s3.bucket.list()
+        self._s3 = get_storage_class('onadata.libs.utils.extended_s3boto_storage.ExtendedS3BotoStorage')()
+        all_files = self._s3.bucket.list(prefix="oleger1")
         size_to_reclaim = 0
         orphans = 0
 
@@ -151,13 +151,14 @@ def handle(self, *args, **kwargs):
     def delete(self, file_object):
         try:
             print("File {} does not exist in DB".format(file_object.name).encode('utf-8'))
-            file_object.delete()
+            self._s3.delete_all(file_object.name)
         except Exception as e:
             print("ERROR - Could not delete file {} - Reason {}".format(
                 file_object.name,
                 str(e)))
 
-    def sizeof_fmt(self, num, suffix='B'):
+    @staticmethod
+    def sizeof_fmt(num, suffix='B'):
         for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
             if abs(num) < 1024.0:
                 return "%3.1f%s%s" % (num, unit, suffix)

onadata/libs/utils/extended_s3boto_storage.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from storages.backends.s3boto import S3BotoStorage
+
+
+class ExtendedS3BotoStorage(S3BotoStorage):
+
+    def delete_all(self, name):
+        """
+        Delete the key object and all its versions
+        :param name: str. S3 key (i.e. path to the file)
+        """
+        name = self._normalize_name(self._clean_name(name))
+        self.bucket.delete_key(self._encode_name(name))
+
+        # Delete all previous versions
+        for versioned_key in self.bucket.list_versions(prefix=name):
+            self.bucket.delete_key(versioned_key.name, version_id=versioned_key.version_id)