Merge pull request #3 from cta-observatory/update-ctapipe-v0.12

Update MAGIC+LST1 analysis scripts to ctapipe v0.12

.gitignore

@@ -12,7 +12,6 @@ build/
 magic_cta_pipe.egg-info/
 magicctapipe/_version.py
 
-
 # Standard files
 *.png
 *.pdf
@@ -29,3 +28,4 @@ TODO.txt
 test.ipynb
 .ipynb_checkpoints
 *.tmp
+*.stats

README.md

@@ -1,10 +1,31 @@
-# ICRR-MPP analysis pipeline for MAGIC and LST data
+# magic-cta-pipe
 
-This repository contains the scripts needed to perform MAGIC+LST analysis with ctapipe.
+Repository for the analysis of MAGIC and MAGIC+LST1 data, based on [*ctapipe*](https://github.com/cta-observatory/ctapipe).
+
+* Code: https://github.com/cta-observatory/magic-cta-pipe
 
 ### Note: This package is under heavy development. Usage at your own risk, it is recommended to contact one of the latest committers if you plan to use this package.
 
+# Installation for users
+
+*magic-cta-pipe* and its dependencies may be installed using the *Anaconda* or *Miniconda* package system. We recommend creating a conda virtual environment
+first, to isolate the installed version and dependencies from your master environment (this is optional).
+
+The following command will set up a conda virtual environment, add the necessary package channels, and install *magic-cta-pipe* and its dependencies::
+
+    wget https://raw.githubusercontent.com/cta-observatory/magic-cta-pipe/master/environment.yml
+    conda env create -n magic-lst1 -f environment.yml
+    conda activate magic-lst1
+    pip install magic-cta-pipe
+
+# Available scripts
+
+*magic-cta-pipe* scripts to perform the analysis of MAGIC only and MAGIC+LST1 data. Within the *scripts* folder we have the subdirectories:
+
+* *magic*: scripts for the analysis of MAGIC data (currently under heavy restructuring in order to be used with ctapipe v0.12)
+* *lst1_magic_real*: scripts for the MAGIC+LST1 analysis (updated to be used with ctapipe v0.12)
 
+<!--
 A brief description:
 1. `config/CrabNebula.yaml`: an example of the configuration file, used by all the scripts.
 2. `config/magic-cta-pipe_config_stereo.yaml`: an example of the configuration file for stereo analysis.
@@ -247,3 +268,4 @@ $ python make_event_lists.py --config=config.yaml
 ```
 
 If you used the `--stereo` option for the previous scripts, then also `make_event_lists.py` should be called with the `--stereo` option.
+-->

environment.yml

@@ -0,0 +1,54 @@
+# A conda environment with all useful package for ctapipe developers
+name: magic-lst1
+channels:
+  - default
+  - cta-observatory
+dependencies:
+  - python=3.7
+  - pip
+  - astropy
+  - black
+  - bokeh=1
+  - conda-forge::nbsphinx
+  - conda-forge::ctapipe=0.12
+  - conda-forge::gammapy=0.18.2
+  - cython
+  - graphviz
+  - h5py
+  - iminuit>=2
+  - ipython
+  - ipywidgets
+  - joblib
+  - jupyter
+  - matplotlib
+  - numba
+  - numpy>=1.17
+  - numpydoc
+  - pandas
+  - pre-commit
+  - psutil
+  - pytables
+  - pytest
+  - pytest-cov
+  - pytest-runner
+  - pyyaml
+  - scikit-learn
+  - scipy
+  - setuptools
+  - sphinx=3.5
+  - sphinx-automodapi
+  - sphinx_rtd_theme
+  - tqdm
+  - traitlets
+  - vitables
+  - wheel
+  - xz
+  - zlib
+  - zstandard
+  - conda-forge::eventio>=1.5.0
+  - conda-forge::corsikaio
+  - pip:
+      - ctapipe_io_magic~=0.4.0
+      - uproot~=4.1
+      - ctaplot~=0.5.3
+      - pyirf~=0.5.0

magicctapipe/__init__.py

@@ -12,4 +12,3 @@
     "utils",
     "__version__",
 ]
-

magicctapipe/_dev_version.py

@@ -7,4 +7,3 @@
     version = get_version(root="..", relative_to=__file__)
 except Exception as e:
     raise ImportError(f"setuptools_scm broken or not installed: {e}")
-

magicctapipe/config/CrabNebula.yaml

@@ -36,21 +36,26 @@ data_files:
                input_mask: "../../../Data/MAGIC/CrabNebula/1.Calibrated/20*_M2_*root"
                hillas_output: "../../../Data/MAGIC/CrabNebula/ctapipe/2.Hillas/iv_hillas_m2.h5"
                reco_output: "../../../Data/MAGIC/CrabNebula/ctapipe/3.Reco/reco_m2.h5"
-
 
 image_cleaning:
     magic:
-        charge_thresholds:
-            picture_thresh: 6
-            boundary_thresh: 3.5
-            min_number_picture_neighbors: 1
+        picture_thresh: 6
+        boundary_thresh: 3.5
+        min_number_picture_neighbors: 1
+        # 1.5 slices x 1.64 GHz
+        max_time_diff: 2.46
+        # 4.5 slices x 1.64 GHz
+        max_time_off: 7.38
+        min_number_neighbors: 3
+        usetime : True
+        usesum : True
+        findhotpixels : True
 
-        time_thresholds:
-            # 1.5 slices x 1.64 GHz
-            max_time_diff: 2.46
-            # 4.5 slices x 1.64 GHz
-            max_time_off: 7.38
-            min_number_neighbors: 1
+bad_pixels:
+    magic:
+        pedestalLevel: 400
+        pedestalLevelVariance: 4.5
+        pedestalType: "FromExtractorRndm"
 
 energy_rf:
     save_name: "RFs/energy_rf.joblib"

magicctapipe/reco/image.py

@@ -4,7 +4,6 @@
 
 import astropy.units as u
 
-
 from ctapipe.image import (
     leakage,
     hillas_parameters,

magicctapipe/scripts/lst1_magic_real/__init__.py

@@ -1,7 +1,6 @@
-from .magic_data_cal_to_dl1 import *
-from .lst1_magic_data_check_pointings import *
-from .lst1_magic_data_event_coincidence import *
-from .lst1_magic_stereo_reco import *
+from .lst1_magic_dl1_to_dl2 import *
+from .lst1_magic_event_coincidence import *
 from .lst1_magic_mc_dl0_to_dl1 import *
+from .lst1_magic_stereo_reco import *
 from .lst1_magic_train_rfs import *
-from .lst1_magic_dl1_to_dl2 import *
+from .magic_data_cal_to_dl1 import *