-
Notifications
You must be signed in to change notification settings - Fork 0
/
make_lyra_event_list1.py
249 lines (219 loc) · 9.41 KB
/
make_lyra_event_list1.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
# Outline of code
#-------Prepare data-------
# 1. Create LYRALightcurve object for given time interval from level 3
# fits files (1-min averaged).
# 2. Download LYTAF file from server. Add keyword to allow user to
# define local file.
# 3. Remove data during UV occulations, offpoints, calibrations, and
# large angle rotations. (Also remove recovery times?)
# 4. Download Ingolf's comb-like masks. Add keyword so they can be
# defined locally.
# 5. Use the masks to remove comb-like features.
#------Find possible start times flares--------
# 6. Take derivative of time series.
# 7. Take where derivative is positive.
# 8. Take where derivative is negative.
# 9. Copy time array and shift by 4 elements.
# 10. Subtract one from other and find where result = 4 minutes.
# 11. Define earlier time corresponding to previous step as preliminary
# start times.
#-----Find end and peak times---------
# 12. Loop through start times.
# 13. Find next time where derivative in negative.
# 14. While end time criterion not reached, loop through following times
# where derivative is negative and find when end criterion is
# reached.
# 15. Define peak time as time of maximum between start and end time.
#-----Look for flares during decay phase------
# 16. If there are possible start times between peak and end time,
# repeat steps 13-16.
from __future__ import absolute_import
from __future__ import division
import os.path
from datetime import datetime
from sunpy.time import parse_time
import sqlite3
from itertools import chain
# Define location of LYRA Annotation Files
LYTAF_PATH = os.path.join(os.path.curdir, "data")
def extract_combined_lytaf(tstart, tend,
combine_files=["lyra", "manual", "ppt", "science"]):
"""
Extracts combined lytaf file for given time range.
Given a time range defined by start_time and end_time, this
function extracts the segments of each LYRA annotation file and
combines them.
Parameters
----------
start_time : datetime object or string
Start time of period for which annotation file is
required.
end_time : datetime object or string
End time of period for which annotation file is
required.
combine_files : (optional) list of strings
States which LYRA annotation files are to be
combined.
Default is all four, i.e. lyra, manual, ppt,
science.
See Notes section for an explanation of each.
Returns
-------
lytaf : dictionary containing the various parameters stored in the
LYTAF files.
Notes
-----
There are four LYRA annotation files which mark different types of
events or artifacts in the data. They are named
annotation_suffix.db where suffix is a variable equalling either
lyra, manual, ppt, or science.
annotation_lyra.db : contains entries regarding possible effects to
the data due to normal operation of LYRA
instrument.
annotation_manual.db : contains entries regarding possible effects
to the data due to unusual or manually
logged events.
annotation_ppt.db : contains entries regarding possible effects to
the data due to pointing or positioning of
PROBA2.
annotation_science.db : contains events in the data scientifically
interesting, e.g. flares.
References
----------
Further documentation: http://proba2.oma.be/data/TARDIS
Examples
--------
"""
# Check inputs
# Check start_time is a date string or datetime object
if type(tstart) is str:
tstart = parse_time(tstart)
if type(tstart) is not datetime.datetime:
raise TypeError("start_time must be a date string or datetime object")
# Check start_time is a date string or datetime object
if type(tend) is str:
tend = parse_time(tend)
if type(tend) is not datetime.datetime:
raise TypeError("end_time must be a date string or datetime object")
# Check combine_files contains correct inputs
if not all(suffix in ["lyra", "manual", "ppt", "science"]
for suffix in combine_files):
raise TypeError("Elements in combine_files must be strings equalling "
"'lyra', 'manual', 'ppt', or 'science'.")
# Remove any duplicates from combine_files input
combine_files = list(set(combine_files))
combine_files.sort()
# Convert input times to UNIX timestamp format since this is the
# time format in the annotation files
tstart_uts = tstart - datetime(1973,1,1).total_seconds()
tend_uts = tend - datetime(1973,1,1).total_seconds()
# Access annotation files
# Define list to hold data from event tables in annotation files.
# First element is defined as empty list for convenience in for loop below.
# This will be deleted afterwards.
event_rows = [[]]
for i, suffix in enumerate(combine_files):
# Open SQLITE3 annotation files
connection = sqlite3.connect(os.path.join(LYTAF_PATH, "annotation_"
+ suffix + ".db"))
# Create cursor to manipulate data in annotation file
cursor = connection.cursor()
# Select and extract the data from event table within file within
# given time range
cursor.execute("select insertion_time, begin_time, reference_time, "
"end_time, eventType_id from event where begin_time >= "
+ tstart_uts + " and begin_time <= " + tend_uts)
event_rows.append(cursor.fetchall())
# Select and extract the event types from eventType table
cursor.row_factory = sqlite3.Row
cursor.execute("select * from eventType")
eventType_rows = cursor.fetchall()
eventType_id = []
eventType_type = []
eventType_definition = []
for j, row in enumerate(eventType_rows):
eventType_id.append(row["id"])
eventType_type.append(row["type"])
eventType_definition.append(row["definition"])
# Replace event type IDs with event type description
for j, row in enumerate(event_rows[i+1]):
_id = eventType_id.index(event_rows[i+1][j][4])
event_rows[i+1][j] = (event_rows[i+1][j][0], event_rows[i+1][j][1],
event_rows[i+1][j][2], event_rows[i+1][j][3],
eventType_rows[_id][1],
eventType_rows[_id][2])
# Close file
cursor.close()
connection.close()
# Delete empty string as first element in event_rows
del(event_rows[0])
# Format and Output data
# Make event_rows a unified list rather than a list of lists
# where each sublist corresponds to an annotation file.
event_rows = list(chain.from_iterable(event_rows))
# Sort arrays in order of increasing start time.
event_rows.sort(key=lambda tup: tup[1])
# Convert times to datetime objects
# INSERT CODE HERE
# Create dictionary to hold results
lytaf = {"insertion_time": [],
"begin_time": [],
"reference_time": [],
"end_time": [],
"event_type": [],
"event_definition": []
}
# Put results into dictionary using for loop
for i, row in enumerate(event_rows):
lytaf["insertion_time"].append(datetime.fromtimestamp(
event_rows[i][0]))
lytaf["begin_time"].append(datetime.fromtimestamp(event_rows[i][1]))
lytaf["reference_time"].append(datetime.fromtimestamp(
event_rows[i][2]))
lytaf["end_time"].append(datetime.fromtimestamp(event_rows[i][3]))
lytaf["event_type"].append(event_rows[i][4])
lytaf["event_definition"].append(event_rows[i][5])
#return event_rows, eventType_rows
return lytaf
def find_lyra_events(flux, time):
"""
Finds events in a times series satisfying LYRA event definitions.
This function finds events in an input time series which satisfy
the LYRA event definitions and returns the start, peak and end
times. For LYRA event definitions, see Notes section of this
docstring.
Parameters
----------
flux : numpy float64 array
Contains flux/irradiance measurements
time : sunpy datetime object or string array/list
Contains measurement times corresponding to each element in
flux. Must be same length as flux.
Returns
-------
event_list :
Notes
-----
Start time:
1) There must be a continuous increase in 1-minute-averaged data
over 4 minutes.
2) The flux in the 4th minute must be at least 1.4 times the flux
in the first minute.
End time:
1) The end time is when the flux falls to half-way between the peak
and initial fluxes.
References
---------
http://www.swpc.noaa.gov/ftpdir/indices/events/README
Examples
--------
"""
# Get LYTAF file for given time range
lytaf = extract_combined_lytaf(time[0], time[-1])
# Extract
#Get list of lists
#Replace eventType_id with eventtype for each tuple in each sublist using nested for loops.
#Unpack list of lists
#Order by event start time
#Generate output structure ?Dictionary?
#return