From 5b5b3e8305c9416da1ed806b63601906c751e7ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20L=C3=A9ger?= Date: Wed, 17 Apr 2019 11:31:04 -0400 Subject: [PATCH] Delete all files with their previous versions --- .../management/commands/remove_s3_orphans.py | 9 +++++---- onadata/libs/utils/extended_s3boto_storage.py | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 onadata/libs/utils/extended_s3boto_storage.py diff --git a/onadata/apps/logger/management/commands/remove_s3_orphans.py b/onadata/apps/logger/management/commands/remove_s3_orphans.py index 6f7efdbce..d9f0d61b0 100644 --- a/onadata/apps/logger/management/commands/remove_s3_orphans.py +++ b/onadata/apps/logger/management/commands/remove_s3_orphans.py @@ -73,8 +73,8 @@ def handle(self, *args, **kwargs): Bucket._get_all = _get_all - s3 = get_storage_class('storages.backends.s3boto.S3BotoStorage')() - all_files = s3.bucket.list() + self._s3 = get_storage_class('onadata.libs.utils.extended_s3boto_storage.ExtendedS3BotoStorage')() + all_files = self._s3.bucket.list(prefix="oleger1") size_to_reclaim = 0 orphans = 0 @@ -151,13 +151,14 @@ def handle(self, *args, **kwargs): def delete(self, file_object): try: print("File {} does not exist in DB".format(file_object.name).encode('utf-8')) - file_object.delete() + self._s3.delete_all(file_object.name) except Exception as e: print("ERROR - Could not delete file {} - Reason {}".format( file_object.name, str(e))) - def sizeof_fmt(self, num, suffix='B'): + @staticmethod + def sizeof_fmt(num, suffix='B'): for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']: if abs(num) < 1024.0: return "%3.1f%s%s" % (num, unit, suffix) diff --git a/onadata/libs/utils/extended_s3boto_storage.py b/onadata/libs/utils/extended_s3boto_storage.py new file mode 100644 index 000000000..0b37ed7e6 --- /dev/null +++ b/onadata/libs/utils/extended_s3boto_storage.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from storages.backends.s3boto import S3BotoStorage + + +class ExtendedS3BotoStorage(S3BotoStorage): + + def delete_all(self, name): + """ + Delete the key object and all its versions + :param name: str. S3 key (i.e. path to the file) + """ + name = self._normalize_name(self._clean_name(name)) + self.bucket.delete_key(self._encode_name(name)) + + # Delete all previous versions + for versioned_key in self.bucket.list_versions(prefix=name): + self.bucket.delete_key(versioned_key.name, version_id=versioned_key.version_id)