Delete unzipped raw data ASCII

holukas · Apr 2, 2023 · e62bf84 · e62bf84
1 parent 7bc7659
commit e62bf84
Show file tree

Hide file tree

Showing 7 changed files with 169 additions and 59 deletions.
diff --git a/.idea/csv-editor.xml b/.idea/csv-editor.xml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,15 @@
 
 ![](images/logo_FLUXRUN1_256px.png)
 
+## v1.3.0 - 2 Apr 2023
+
+- New setting in `Output` > `After Processing` > `Delete uncompressed raw data ASCII`:
+  if selected, the uncompressed raw data files (ASCII) are deleted after finished flux
+  calculations. The *compressed* raw data ASCII files are not affected by this setting.
+  Background: for EddyPro flux calculations, the zipped files are uncompressed to regular
+  ASCII files. These uncompressed files are not required for long-term storage because they
+  take up a lot of disk space. For storage, we keep the zipped (compressed) ASCII files.
+
 ## v1.2.0 - 20 Mar 2023
 
 - Now using `poetry` as dependency manager

diff --git a/fluxrun/fluxrun.py b/fluxrun/fluxrun.py
@@ -108,7 +108,16 @@ def run(self):
         if rp_process_status == 0 and found_full_output == False:
             fcc_process_status = self.run_eddypro_cmd(cmd='eddypro_fcc.exe')  # execute exe todo for linux and osx
 
-        # Plot summary
+        self._plot_summary()
+        self._delete_uncompressed_ascii_files()
+
+        self.logger.info("\n\n\n")
+        self.logger.info("=" * 60)
+        self.logger.info("FluxRun finished.")
+        self.logger.info("=" * 60)
+
+    def _plot_summary(self):
+        """Generate summary plots"""
         found_full_output, filepath_full_output = \
             ops.file.check_if_file_in_folder(search_str='*_full_output_*.csv',
                                              folder=self.settings_dict['_dir_out_run_eddypro_results'])
@@ -117,14 +126,36 @@ def run(self):
                 file_to_plot=filepath_full_output,
                 destination_folder=self.settings_dict['_dir_out_run_plots_summary'],
                 logger=self.logger).run()
-
         else:
             self.logger.info("(!)WARNING No *_full_output_* file was found. Skipping summary plots.")
 
-        self.logger.info("\n\n\n")
-        self.logger.info("=" * 60)
-        self.logger.info("FluxRun finished.")
-        self.logger.info("=" * 60)
+
+    def _delete_uncompressed_ascii_files(self):
+        """Delete uncompressed (unzipped) ASCII files that were used for flux processing"""
+        if int(self.settings_dict['delete_uncompressed_ascii_after_processing']) == 1:
+            uncompressed_ascii_files = ops.file.SearchAll(
+                settings_dict=self.settings_dict,
+                logger=self.logger,
+                search_in_dir=self.settings_dict['_dir_out_run_rawdata_ascii_files'],
+                search_uncompressed=True) \
+                .keep_valid_files()
+
+            # Convert to list
+            deletepaths = []
+            for filename, filepath in uncompressed_ascii_files.items():
+                deletepaths.append(filepath)
+
+            # Keep the last file
+            deletepaths = list(deletepaths)[:-1]
+
+            # Make sure there are CSVs only
+            deletelist = []
+            [deletelist.append(x) for x in deletepaths if x.suffix == '.csv']
+
+            # Delete files
+            for filepath in deletelist:
+                self.logger.info(f"Deleting uncompressed (unzipped) ASCII file: {filepath}) ...")
+                os.remove(filepath)
 
     def run_eddypro_cmd(self, cmd: str):
         """Run eddypro_rp.exe or eddypro_fcc.exe"""
@@ -266,6 +297,9 @@ def get_settings_from_gui(self):
         self.update_dict_key(key='plot_summary',
                              new_val='1' if self.chk_output_plots_summary.isChecked() else '0')
 
+        self.update_dict_key(key='delete_uncompressed_ascii_after_processing',
+                             new_val='1' if self.chk_output_afterprocessing_delete_ascii_rawdata.isChecked() else '0')
+
     def set_gui_combobox(self, combobox, find_text):
         idx = combobox.findText(find_text, qtc.Qt.MatchContains)
         if idx >= 0:
@@ -309,6 +343,9 @@ def show_settings_in_gui(self):
         self.set_gui_checkbox(checkbox=self.chk_output_plots_summary,
                               state=self.settings_dict['plot_summary'])
 
+        self.set_gui_checkbox(checkbox=self.chk_output_afterprocessing_delete_ascii_rawdata,
+                              state=self.settings_dict['delete_uncompressed_ascii_after_processing'])
+
     def link(self, link_str):
         """Call hyperlink from label, opens in browser"""
         qtg.QDesktopServices.openUrl(qtc.QUrl(link_str))

diff --git a/fluxrun/gui/gui.py b/fluxrun/gui/gui.py
@@ -137,7 +137,14 @@ def add_section_output(self):
         self.chk_output_plots_summary = \
             gui_elements.add_checkbox_to_grid(label='Summary (Flux Processing)', grid=grid, row=7)
 
-        grid.setRowStretch(8, 1)
+        # After processing
+        header_output_plots = qtw.QLabel('After Processing')
+        header_output_plots.setProperty('labelClass', 'header_2')
+        grid.addWidget(header_output_plots, 8, 0, 1, 1)
+        self.chk_output_afterprocessing_delete_ascii_rawdata = \
+            gui_elements.add_checkbox_to_grid(label='Delete uncompressed raw data ASCII', grid=grid, row=9)
+
+        grid.setRowStretch(10, 1)
         section.setLayout(grid)
         return section
 

diff --git a/fluxrun/settings/FluxRun.settings b/fluxrun/settings/FluxRun.settings
@@ -1,4 +1,4 @@
-_run_id=FR-20230320-224726
+_run_id=FR-20230402-204148
 
 
 # INSTRUMENTS
@@ -12,54 +12,57 @@ site=CH-FRU
 # ==========
 
 ## Raw Data Source Folder
-rawdata_indir=L:/Sync/luhk_work/20 - CODING/23 - FLUX/fluxrun/_tests/XXX2
+rawdata_indir=F:/Sync/luhk_work/20 - CODING/23 - FLUX/fluxrun/_tests/XXX2
 rawdata_file_compression=gzip
 
 ## Time Range
 rawdata_start_date=2022-01-01 00:01
-rawdata_end_date=2022-01-31 23:59
+rawdata_end_date=2022-12-31 23:59
 
 ## File Settings
 rawdata_filename_datetime_format=yyyymmddHHMM
 _sitefiles_search_str=CH-FRU_*.csv.gz
 _sitefiles_parse_str=CH-FRU_%Y%m%d%H%M.csv.gz
 
 ## EddyPro
-path_selected_eddypro_processing_file=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_2022_[IRGA75]_R350-A+IRGA75-A.eddypro
-_path_used_eddypro_processing_file=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-0_eddypro_flux_calculations\ini\processing.eddypro
-_path_found_eddypro_metadata_file=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_2022_[IRGA75]_R350-A+IRGA75-A.eddypro
-_path_used_eddypro_metadata_file=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-0_eddypro_flux_calculations\ini\CH-FRU_2022_[IRGA75]_R350-A+IRGA75-A.metadata
-_path_used_eddypro_app_rp=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-0_eddypro_flux_calculations\bin\eddypro_rp.exe
-_path_used_eddypro_app_fcc=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-0_eddypro_flux_calculations\bin\eddypro_fcc.exe
+path_selected_eddypro_processing_file=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_2022_[IRGA75]_R350-A+IRGA75-A.eddypro
+_path_used_eddypro_processing_file=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-0_eddypro_flux_calculations\ini\processing.eddypro
+_path_found_eddypro_metadata_file=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_2022_[IRGA75]_R350-A+IRGA75-A.eddypro
+_path_used_eddypro_metadata_file=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-0_eddypro_flux_calculations\ini\CH-FRU_2022_[IRGA75]_R350-A+IRGA75-A.metadata
+_path_used_eddypro_app_rp=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-0_eddypro_flux_calculations\bin\eddypro_rp.exe
+_path_used_eddypro_app_fcc=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-0_eddypro_flux_calculations\bin\eddypro_fcc.exe
 
 # OUTPUT
 # ======
 
 ## Output Folder
-dir_out=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2
-_dir_out_run=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726
-_dir_out_run_log=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\0_log
-_dir_out_run_rawdata_ascii_files=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\1-0_rawdata_files_ascii
-_dir_used_rawdata_ascii_files_eddypro_data_path=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\1-0_rawdata_files_ascii
-_dir_out_run_plots_availability_rawdata=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\1-1_rawdata_plots_availability
-_dir_out_run_plots_aggregates_rawdata=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\1-2_rawdata_plots_aggregates
-_dir_out_run_eddypro=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-0_eddypro_flux_calculations
-_dir_out_run_eddypro_ini=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-0_eddypro_flux_calculations\ini
-_dir_out_run_eddypro_bin=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-0_eddypro_flux_calculations\bin
-_dir_out_run_eddypro_results=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-0_eddypro_flux_calculations\results
-_dir_out_run_plots_summary=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230320-224726\2-1_eddypro_flux_calculations_summary_plots
+dir_out=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2
+_dir_out_run=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148
+_dir_out_run_log=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\0_log
+_dir_out_run_rawdata_ascii_files=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\1-0_rawdata_files_ascii
+_dir_used_rawdata_ascii_files_eddypro_data_path=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\1-0_rawdata_files_ascii
+_dir_out_run_plots_availability_rawdata=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\1-1_rawdata_plots_availability
+_dir_out_run_plots_aggregates_rawdata=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\1-2_rawdata_plots_aggregates
+_dir_out_run_eddypro=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-0_eddypro_flux_calculations
+_dir_out_run_eddypro_ini=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-0_eddypro_flux_calculations\ini
+_dir_out_run_eddypro_bin=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-0_eddypro_flux_calculations\bin
+_dir_out_run_eddypro_results=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-0_eddypro_flux_calculations\results
+_dir_out_run_plots_summary=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\_tests\XXX2\CH-FRU_FR-20230402-204148\2-1_eddypro_flux_calculations_summary_plots
 
 ## Plots
 plot_availability_rawdata=1
 plot_aggregates_rawdata=1
 plot_summary=1
 
+## After Processing
+delete_uncompressed_ascii_after_processing=1
+
 
 # _SCRIPT
 # =======
 
 ## General
-_dir_root=L:\Sync\luhk_work\20 - CODING\23 - FLUX
-_dir_fluxrun=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun
-_dir_script=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\fluxrun
-_dir_settings=L:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\fluxrun\settings
+_dir_root=F:\Sync\luhk_work\20 - CODING\23 - FLUX
+_dir_fluxrun=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun
+_dir_script=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\fluxrun
+_dir_settings=F:\Sync\luhk_work\20 - CODING\23 - FLUX\fluxrun\fluxrun\settings