Merge pull request #1423 from bug-or-feature/bug_1364_parquet_capital

improved parquet capital
robcarver17 · Oct 25, 2024 · 8cf5e70 · 8cf5e70
2 parents b264d6d + 44490b1
commit 8cf5e70
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 10 deletions.
diff --git a/sysdata/parquet/parquet_access.py b/sysdata/parquet/parquet_access.py
@@ -1,5 +1,6 @@
 import os
 import pandas as pd
+from syscore.exceptions import missingFile
 from syscore.fileutils import (
     files_with_extension_in_pathname,
     resolve_path_and_filename_for_package,
@@ -33,7 +34,10 @@ def delete_data_given_data_type_and_identifier(
         filename = self._get_filename_given_data_type_and_identifier(
             data_type=data_type, identifier=identifier
         )
-        os.remove(filename)
+        try:
+            os.remove(filename)
+        except FileNotFoundError:
+            raise missingFile(f"File '{filename}' does not exist")
 
     def write_data_given_data_type_and_identifier(
         self, data_to_write: pd.DataFrame, data_type: str, identifier: str

diff --git a/sysdata/parquet/parquet_capital.py b/sysdata/parquet/parquet_capital.py
@@ -1,4 +1,4 @@
-from syscore.exceptions import missingData
+from syscore.exceptions import missingData, missingFile
 from sysdata.production.capital import capitalData
 
 CAPITAL_COLLECTION = "capital"
@@ -45,9 +45,15 @@ def get_capital_pd_df_for_strategy(self, strategy_name: str) -> pd.DataFrame:
         return pd_df
 
     def _delete_all_capital_for_strategy_no_checking(self, strategy_name: str):
-        self.parquet.delete_data_given_data_type_and_identifier(
-            data_type=CAPITAL_COLLECTION, identifier=strategy_name
-        )
+        try:
+            self.parquet.delete_data_given_data_type_and_identifier(
+                data_type=CAPITAL_COLLECTION, identifier=strategy_name
+            )
+        except missingFile as mf:
+            self.log.warning(
+                f"Deleting old capital failed: {mf} - "
+                f"this is ok if setting up capital for the first time"
+            )
 
     def update_capital_pd_df_for_strategy(
         self, strategy_name: str, updated_capital_df: pd.DataFrame

diff --git a/sysinit/transfer/backup_arctic_to_parquet.py b/sysinit/transfer/backup_arctic_to_parquet.py
@@ -448,11 +448,16 @@ def backup_capital(data):
         strategy_capital_data = data.arctic_capital.get_capital_pd_df_for_strategy(
             strategy_name
         )
-        parquet_data = data.parquet_capital.get_capital_pd_df_for_strategy(
-            strategy_name
-        )
-        if len(parquet_data) > strategy_capital_data:
-            data.log.warning("More parquet data, skipping")
+        try:
+            parquet_data = data.parquet_capital.get_capital_pd_df_for_strategy(
+                strategy_name
+            )
+        except missingData:
+            parquet_data = []
+
+        if len(parquet_data) >= len(strategy_capital_data):
+            data.log.debug(f"No backup needed for '{strategy_name}', skipping")
+            continue
 
         data.parquet_capital.update_capital_pd_df_for_strategy(
             strategy_name=strategy_name, updated_capital_df=strategy_capital_data