##// END OF EJS Templates
schain-jc
RSS
Add reading online mode for BLTR parameters
Juan C. Espinoza -
r1215:785b27922a3d
parent child
Show More
Side by Side Unified
Context file:
r1215:785b27922a3d Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,368 +1,409
1 '''
1 '''
2 Created on Nov 9, 2016
2 Created on Nov 9, 2016
3
3
4 @author: roj- LouVD
4 @author: roj- LouVD
5 '''
5 '''
6
6
7
7
8 import os
8 import os
9 import sys
9 import sys
10 import time
10 import time
11 import glob
11 import glob
12 import datetime
12 import datetime
13
13
14 import numpy
14 import numpy
15
15
16 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
16 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
17 from schainpy.model.data.jrodata import Parameters
17 from schainpy.model.data.jrodata import Parameters
18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
19 from schainpy.utils import log
19 from schainpy.utils import log
20
20
21 FILE_HEADER_STRUCTURE = numpy.dtype([
21 FILE_HEADER_STRUCTURE = numpy.dtype([
22 ('FMN', '<u4'),
22 ('FMN', '<u4'),
23 ('nrec', '<u4'),
23 ('nrec', '<u4'),
24 ('fr_offset', '<u4'),
24 ('fr_offset', '<u4'),
25 ('id', '<u4'),
25 ('id', '<u4'),
26 ('site', 'u1', (32,))
26 ('site', 'u1', (32,))
27 ])
27 ])
28
28
29 REC_HEADER_STRUCTURE = numpy.dtype([
29 REC_HEADER_STRUCTURE = numpy.dtype([
30 ('rmn', '<u4'),
30 ('rmn', '<u4'),
31 ('rcounter', '<u4'),
31 ('rcounter', '<u4'),
32 ('nr_offset', '<u4'),
32 ('nr_offset', '<u4'),
33 ('tr_offset', '<u4'),
33 ('tr_offset', '<u4'),
34 ('time', '<u4'),
34 ('time', '<u4'),
35 ('time_msec', '<u4'),
35 ('time_msec', '<u4'),
36 ('tag', 'u1', (32,)),
36 ('tag', 'u1', (32,)),
37 ('comments', 'u1', (32,)),
37 ('comments', 'u1', (32,)),
38 ('lat', '<f4'),
38 ('lat', '<f4'),
39 ('lon', '<f4'),
39 ('lon', '<f4'),
40 ('gps_status', '<u4'),
40 ('gps_status', '<u4'),
41 ('freq', '<u4'),
41 ('freq', '<u4'),
42 ('freq0', '<u4'),
42 ('freq0', '<u4'),
43 ('nchan', '<u4'),
43 ('nchan', '<u4'),
44 ('delta_r', '<u4'),
44 ('delta_r', '<u4'),
45 ('nranges', '<u4'),
45 ('nranges', '<u4'),
46 ('r0', '<u4'),
46 ('r0', '<u4'),
47 ('prf', '<u4'),
47 ('prf', '<u4'),
48 ('ncoh', '<u4'),
48 ('ncoh', '<u4'),
49 ('npoints', '<u4'),
49 ('npoints', '<u4'),
50 ('polarization', '<i4'),
50 ('polarization', '<i4'),
51 ('rx_filter', '<u4'),
51 ('rx_filter', '<u4'),
52 ('nmodes', '<u4'),
52 ('nmodes', '<u4'),
53 ('dmode_index', '<u4'),
53 ('dmode_index', '<u4'),
54 ('dmode_rngcorr', '<u4'),
54 ('dmode_rngcorr', '<u4'),
55 ('nrxs', '<u4'),
55 ('nrxs', '<u4'),
56 ('acf_length', '<u4'),
56 ('acf_length', '<u4'),
57 ('acf_lags', '<u4'),
57 ('acf_lags', '<u4'),
58 ('sea_to_atmos', '<f4'),
58 ('sea_to_atmos', '<f4'),
59 ('sea_notch', '<u4'),
59 ('sea_notch', '<u4'),
60 ('lh_sea', '<u4'),
60 ('lh_sea', '<u4'),
61 ('hh_sea', '<u4'),
61 ('hh_sea', '<u4'),
62 ('nbins_sea', '<u4'),
62 ('nbins_sea', '<u4'),
63 ('min_snr', '<f4'),
63 ('min_snr', '<f4'),
64 ('min_cc', '<f4'),
64 ('min_cc', '<f4'),
65 ('max_time_diff', '<f4')
65 ('max_time_diff', '<f4')
66 ])
66 ])
67
67
68 DATA_STRUCTURE = numpy.dtype([
68 DATA_STRUCTURE = numpy.dtype([
69 ('range', '<u4'),
69 ('range', '<u4'),
70 ('status', '<u4'),
70 ('status', '<u4'),
71 ('zonal', '<f4'),
71 ('zonal', '<f4'),
72 ('meridional', '<f4'),
72 ('meridional', '<f4'),
73 ('vertical', '<f4'),
73 ('vertical', '<f4'),
74 ('zonal_a', '<f4'),
74 ('zonal_a', '<f4'),
75 ('meridional_a', '<f4'),
75 ('meridional_a', '<f4'),
76 ('corrected_fading', '<f4'), # seconds
76 ('corrected_fading', '<f4'), # seconds
77 ('uncorrected_fading', '<f4'), # seconds
77 ('uncorrected_fading', '<f4'), # seconds
78 ('time_diff', '<f4'),
78 ('time_diff', '<f4'),
79 ('major_axis', '<f4'),
79 ('major_axis', '<f4'),
80 ('axial_ratio', '<f4'),
80 ('axial_ratio', '<f4'),
81 ('orientation', '<f4'),
81 ('orientation', '<f4'),
82 ('sea_power', '<u4'),
82 ('sea_power', '<u4'),
83 ('sea_algorithm', '<u4')
83 ('sea_algorithm', '<u4')
84 ])
84 ])
85
85
86 @MPDecorator
86 @MPDecorator
87 class BLTRParamReader(JRODataReader, ProcessingUnit):
87 class BLTRParamReader(JRODataReader, ProcessingUnit):
88 '''
88 '''
89 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
89 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
90 '''
90 '''
91
91
92 ext = '.sswma'
92 ext = '.sswma'
93
93
94 def __init__(self):
94 def __init__(self):
95
95
96 ProcessingUnit.__init__(self)
96 ProcessingUnit.__init__(self)
97
97
98 self.dataOut = Parameters()
98 self.dataOut = Parameters()
99 self.counter_records = 0
99 self.counter_records = 0
100 self.flagNoMoreFiles = 0
100 self.flagNoMoreFiles = 0
101 self.isConfig = False
101 self.isConfig = False
102 self.filename = None
102 self.filename = None
103
103
104 def setup(self,
104 def setup(self,
105 path=None,
105 path=None,
106 startDate=None,
106 startDate=None,
107 endDate=None,
107 endDate=None,
108 ext=None,
108 ext=None,
109 startTime=datetime.time(0, 0, 0),
109 startTime=datetime.time(0, 0, 0),
110 endTime=datetime.time(23, 59, 59),
110 endTime=datetime.time(23, 59, 59),
111 timezone=0,
111 timezone=0,
112 status_value=0,
112 status_value=0,
113 **kwargs):
113 **kwargs):
114 self.path = path
114 self.path = path
115 self.startDate = startDate
115 self.startDate = startDate
116 self.endDate = endDate
116 self.endDate = endDate
117 self.startTime = startTime
117 self.startTime = startTime
118 self.endTime = endTime
118 self.endTime = endTime
119 self.status_value = status_value
119 self.status_value = status_value
120 self.datatime = datetime.datetime(1900,1,1)
120 self.datatime = datetime.datetime(1900,1,1)
121 self.delay = kwargs.get('delay', 10)
122 self.online = kwargs.get('online', False)
123 self.nTries = kwargs.get('nTries', 3)
121
124
122 if self.path is None:
125 if self.path is None:
123 raise ValueError("The path is not valid")
126 raise ValueError("The path is not valid")
124
127
125 if ext is None:
128 if ext is None:
126 ext = self.ext
129 ext = self.ext
127
130
128 self.search_files(self.path, startDate, endDate, ext)
131 self.fileList = self.search_files(self.path, startDate, endDate, ext)
129 self.timezone = timezone
132 self.timezone = timezone
130 self.fileIndex = 0
133 self.fileIndex = 0
131
134
132 if not self.fileList:
135 if not self.fileList:
133 raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
136 raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
134 path))
137 path))
135
138
136 self.setNextFile()
139 self.setNextFile()
137
140
141 def search_last_file(self):
142 '''
143 Get last file and add it to the list
144 '''
145
146 for n in range(self.nTries):
147 if n > 0:
148 log.warning(
149 "Waiting %0.2f sec for the next file, try %03d ..." % (self.delay, self.nTries + 1),
150 self.name
151 )
152 time.sleep(self.delay)
153 file_list = os.listdir(self.path)
154 file_list.sort()
155 if file_list:
156 if self.filename:
157 if file_list[-1] not in self.filename:
158 return file_list[-1]
159 else:
160 continue
161 return file_list[-1]
162 return 0
163
138 def search_files(self, path, startDate, endDate, ext):
164 def search_files(self, path, startDate, endDate, ext):
139 '''
165 '''
140 Searching for BLTR rawdata file in path
166 Searching for BLTR rawdata file in path
141 Creating a list of file to proces included in [startDate,endDate]
167 Creating a list of file to proces included in [startDate,endDate]
142
168
143 Input:
169 Input:
144 path - Path to find BLTR rawdata files
170 path - Path to find BLTR rawdata files
145 startDate - Select file from this date
171 startDate - Select file from this date
146 enDate - Select file until this date
172 enDate - Select file until this date
147 ext - Extension of the file to read
173 ext - Extension of the file to read
148 '''
174 '''
149
175
150 log.success('Searching files in {} '.format(path), 'BLTRParamReader')
176 log.success('Searching files in {} '.format(path), 'BLTRParamReader')
151 foldercounter = 0
177 foldercounter = 0
152 fileList0 = glob.glob1(path, "*%s" % ext)
178 fileList0 = glob.glob1(path, "*%s" % ext)
153 fileList0.sort()
179 fileList0.sort()
154
180
155 self.fileList = []
181 #self.fileList = []
156 self.dateFileList = []
182 #self.dateFileList = []
157
183
158 for thisFile in fileList0:
184 for thisFile in fileList0:
159 year = thisFile[-14:-10]
185 year = thisFile[-14:-10]
160 if not isNumber(year):
186 if not isNumber(year):
161 continue
187 continue
162
188
163 month = thisFile[-10:-8]
189 month = thisFile[-10:-8]
164 if not isNumber(month):
190 if not isNumber(month):
165 continue
191 continue
166
192
167 day = thisFile[-8:-6]
193 day = thisFile[-8:-6]
168 if not isNumber(day):
194 if not isNumber(day):
169 continue
195 continue
170
196
171 year, month, day = int(year), int(month), int(day)
197 year, month, day = int(year), int(month), int(day)
172 dateFile = datetime.date(year, month, day)
198 dateFile = datetime.date(year, month, day)
173
199
174 if (startDate > dateFile) or (endDate < dateFile):
200 if (startDate > dateFile) or (endDate < dateFile):
175 continue
201 continue
176
202
177 self.fileList.append(thisFile)
203 yield thisFile
178 self.dateFileList.append(dateFile)
204 # self.dateFileList.append(dateFile)
179
205
180 return
206 return
181
207
182 def setNextFile(self):
208 def setNextFile(self):
183
209
184 file_id = self.fileIndex
210 if self.online:
185
211 filename = self.search_last_file()
186 if file_id == len(self.fileList):
212 if not filename:
187 self.flagNoMoreFiles = 1
213 self.flagNoMoreFiles = 1
188 return 0
214 return 0
189
215 else:
190 log.success('Opening {}'.format(self.fileList[file_id]), 'BLTRParamReader')
216 try:
191 filename = os.path.join(self.path, self.fileList[file_id])
217 filename = next(self.fileList)
218 except StopIteration:
219 print('Noooo files')
220 self.flagNoMoreFiles = 1
221 return 0
222
223 log.success('Opening {}'.format(filename), 'BLTRParamReader')
192
224
193 dirname, name = os.path.split(filename)
225 dirname, name = os.path.split(filename)
194 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
226 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
195 self.siteFile = name.split('.')[0]
227 self.siteFile = filename.split('.')[0]
196 if self.filename is not None:
228 if self.filename is not None:
197 self.fp.close()
229 self.fp.close()
198 self.filename = filename
230 self.filename = os.path.join(self.path, filename)
199 self.fp = open(self.filename, 'rb')
231 self.fp = open(self.filename, 'rb')
200 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
232 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
201 self.nrecords = self.header_file['nrec'][0]
233 self.nrecords = self.header_file['nrec'][0]
202 self.sizeOfFile = os.path.getsize(self.filename)
234 self.sizeOfFile = os.path.getsize(self.filename)
203 self.counter_records = 0
235 self.counter_records = 0
204 self.flagIsNewFile = 0
236 self.flagIsNewFile = 0
205 self.fileIndex += 1
237 self.fileIndex += 1
206
238
207 return 1
239 return 1
208
240
209 def readNextBlock(self):
241 def readNextBlock(self):
210
242
211 while True:
243 while True:
212 if self.counter_records == self.nrecords:
244 if self.counter_records == self.nrecords:
213 self.flagIsNewFile = 1
245 self.flagIsNewFile = 1
214 if not self.setNextFile():
246 if not self.setNextFile():
215 return 0
247 return 0
216
248
217 self.readBlock()
249 try:
250 pointer = self.fp.tell()
251 self.readBlock()
252 except:
253 if self.waitDataBlock(pointer, 38512) == 1:
254 continue
255 else:
256 if not self.setNextFile():
257 return 0
218
258
219 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
259 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
220 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
260 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
221 log.warning(
261 log.warning(
222 'Reading Record No. {}/{} -> {} [Skipping]'.format(
262 'Reading Record No. {}/{} -> {} [Skipping]'.format(
223 self.counter_records,
263 self.counter_records,
224 self.nrecords,
264 self.nrecords,
225 self.datatime.ctime()),
265 self.datatime.ctime()),
226 'BLTRParamReader')
266 'BLTRParamReader')
227 continue
267 continue
228 break
268 break
229
269
230 log.log('Reading Record No. {}/{} -> {}'.format(
270 log.log('Reading Record No. {}/{} -> {}'.format(
231 self.counter_records,
271 self.counter_records,
232 self.nrecords,
272 self.nrecords,
233 self.datatime.ctime()), 'BLTRParamReader')
273 self.datatime.ctime()), 'BLTRParamReader')
234
274
235 return 1
275 return 1
236
276
237 def readBlock(self):
277 def readBlock(self):
238
278
239 pointer = self.fp.tell()
279 pointer = self.fp.tell()
240 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
280 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
241 self.nchannels = int(header_rec['nchan'][0] / 2)
281 self.nchannels = int(header_rec['nchan'][0] / 2)
242 self.kchan = header_rec['nrxs'][0]
282 self.kchan = header_rec['nrxs'][0]
243 self.nmodes = header_rec['nmodes'][0]
283 self.nmodes = header_rec['nmodes'][0]
244 self.nranges = header_rec['nranges'][0]
284 self.nranges = header_rec['nranges'][0]
245 self.fp.seek(pointer)
285 self.fp.seek(pointer)
246 self.height = numpy.empty((self.nmodes, self.nranges))
286 self.height = numpy.empty((self.nmodes, self.nranges))
247 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
287 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
248 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
288 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
249 self.flagDiscontinuousBlock = 0
289 self.flagDiscontinuousBlock = 0
250
290
251 for mode in range(self.nmodes):
291 for mode in range(self.nmodes):
252 self.readHeader()
292 self.readHeader()
253 data = self.readData()
293 data = self.readData()
254 self.height[mode] = (data[0] - self.correction) / 1000.
294 self.height[mode] = (data[0] - self.correction) / 1000.
255 self.buffer[mode] = data[1]
295 self.buffer[mode] = data[1]
256 self.snr[mode] = data[2]
296 self.snr[mode] = data[2]
257
297
258 self.counter_records = self.counter_records + self.nmodes
298 self.counter_records = self.counter_records + self.nmodes
259
299
260 return
300 return
261
301
262 def readHeader(self):
302 def readHeader(self):
263 '''
303 '''
264 RecordHeader of BLTR rawdata file
304 RecordHeader of BLTR rawdata file
265 '''
305 '''
266
306
267 header_structure = numpy.dtype(
307 header_structure = numpy.dtype(
268 REC_HEADER_STRUCTURE.descr + [
308 REC_HEADER_STRUCTURE.descr + [
269 ('antenna_coord', 'f4', (2, int(self.nchannels))),
309 ('antenna_coord', 'f4', (2, int(self.nchannels))),
270 ('rx_gains', 'u4', (int(self.nchannels),)),
310 ('rx_gains', 'u4', (int(self.nchannels),)),
271 ('rx_analysis', 'u4', (int(self.nchannels),))
311 ('rx_analysis', 'u4', (int(self.nchannels),))
272 ]
312 ]
273 )
313 )
274
314
275 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
315 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
276 self.lat = self.header_rec['lat'][0]
316 self.lat = self.header_rec['lat'][0]
277 self.lon = self.header_rec['lon'][0]
317 self.lon = self.header_rec['lon'][0]
278 self.delta = self.header_rec['delta_r'][0]
318 self.delta = self.header_rec['delta_r'][0]
279 self.correction = self.header_rec['dmode_rngcorr'][0]
319 self.correction = self.header_rec['dmode_rngcorr'][0]
280 self.imode = self.header_rec['dmode_index'][0]
320 self.imode = self.header_rec['dmode_index'][0]
281 self.antenna = self.header_rec['antenna_coord']
321 self.antenna = self.header_rec['antenna_coord']
282 self.rx_gains = self.header_rec['rx_gains']
322 self.rx_gains = self.header_rec['rx_gains']
283 self.time = self.header_rec['time'][0]
323 self.time = self.header_rec['time'][0]
284 dt = datetime.datetime.utcfromtimestamp(self.time)
324 dt = datetime.datetime.utcfromtimestamp(self.time)
285 if dt.date()>self.datatime.date():
325 if dt.date()>self.datatime.date():
286 self.flagDiscontinuousBlock = 1
326 self.flagDiscontinuousBlock = 1
287 self.datatime = dt
327 self.datatime = dt
288
328
289 def readData(self):
329 def readData(self):
290 '''
330 '''
291 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
331 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
292
332
293 Input:
333 Input:
294 status_value - Array data is set to NAN for values that are not equal to status_value
334 status_value - Array data is set to NAN for values that are not equal to status_value
295
335
296 '''
336 '''
297 self.nchannels = int(self.nchannels)
337 self.nchannels = int(self.nchannels)
298
338
299 data_structure = numpy.dtype(
339 data_structure = numpy.dtype(
300 DATA_STRUCTURE.descr + [
340 DATA_STRUCTURE.descr + [
301 ('rx_saturation', 'u4', (self.nchannels,)),
341 ('rx_saturation', 'u4', (self.nchannels,)),
302 ('chan_offset', 'u4', (2 * self.nchannels,)),
342 ('chan_offset', 'u4', (2 * self.nchannels,)),
303 ('rx_amp', 'u4', (self.nchannels,)),
343 ('rx_amp', 'u4', (self.nchannels,)),
304 ('rx_snr', 'f4', (self.nchannels,)),
344 ('rx_snr', 'f4', (self.nchannels,)),
305 ('cross_snr', 'f4', (self.kchan,)),
345 ('cross_snr', 'f4', (self.kchan,)),
306 ('sea_power_relative', 'f4', (self.kchan,))]
346 ('sea_power_relative', 'f4', (self.kchan,))]
307 )
347 )
308
348
309 data = numpy.fromfile(self.fp, data_structure, self.nranges)
349 data = numpy.fromfile(self.fp, data_structure, self.nranges)
310
350
311 height = data['range']
351 height = data['range']
312 winds = numpy.array(
352 winds = numpy.array(
313 (data['zonal'], data['meridional'], data['vertical']))
353 (data['zonal'], data['meridional'], data['vertical']))
314 snr = data['rx_snr'].T
354 snr = data['rx_snr'].T
315
355
316 winds[numpy.where(winds == -9999.)] = numpy.nan
356 winds[numpy.where(winds == -9999.)] = numpy.nan
317 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
357 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
318 snr[numpy.where(snr == -9999.)] = numpy.nan
358 snr[numpy.where(snr == -9999.)] = numpy.nan
319 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
359 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
320 snr = numpy.power(10, snr / 10)
360 snr = numpy.power(10, snr / 10)
321
361
322 return height, winds, snr
362 return height, winds, snr
323
363
324 def set_output(self):
364 def set_output(self):
325 '''
365 '''
326 Storing data from databuffer to dataOut object
366 Storing data from databuffer to dataOut object
327 '''
367 '''
328
368
329 self.dataOut.data_SNR = self.snr
369 self.dataOut.data_SNR = self.snr
330 self.dataOut.height = self.height
370 self.dataOut.height = self.height
331 self.dataOut.data = self.buffer
371 self.dataOut.data = self.buffer
332 self.dataOut.utctimeInit = self.time
372 self.dataOut.utctimeInit = self.time
333 self.dataOut.utctime = self.dataOut.utctimeInit
373 self.dataOut.utctime = self.dataOut.utctimeInit
334 self.dataOut.useLocalTime = False
374 self.dataOut.useLocalTime = False
335 self.dataOut.paramInterval = 157
375 self.dataOut.paramInterval = 157
336 self.dataOut.timezone = self.timezone
376 self.dataOut.timezone = self.timezone
337 self.dataOut.site = self.siteFile
377 self.dataOut.site = self.siteFile
338 self.dataOut.nrecords = self.nrecords / self.nmodes
378 self.dataOut.nrecords = self.nrecords / self.nmodes
339 self.dataOut.sizeOfFile = self.sizeOfFile
379 self.dataOut.sizeOfFile = self.sizeOfFile
340 self.dataOut.lat = self.lat
380 self.dataOut.lat = self.lat
341 self.dataOut.lon = self.lon
381 self.dataOut.lon = self.lon
342 self.dataOut.channelList = list(range(self.nchannels))
382 self.dataOut.channelList = list(range(self.nchannels))
343 self.dataOut.kchan = self.kchan
383 self.dataOut.kchan = self.kchan
344 self.dataOut.delta = self.delta
384 self.dataOut.delta = self.delta
345 self.dataOut.correction = self.correction
385 self.dataOut.correction = self.correction
346 self.dataOut.nmodes = self.nmodes
386 self.dataOut.nmodes = self.nmodes
347 self.dataOut.imode = self.imode
387 self.dataOut.imode = self.imode
348 self.dataOut.antenna = self.antenna
388 self.dataOut.antenna = self.antenna
349 self.dataOut.rx_gains = self.rx_gains
389 self.dataOut.rx_gains = self.rx_gains
350 self.dataOut.flagNoData = False
390 self.dataOut.flagNoData = False
351 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
391 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
352
392
353 def getData(self):
393 def getData(self):
354 '''
394 '''
355 Storing data from databuffer to dataOut object
395 Storing data from databuffer to dataOut object
356 '''
396 '''
357 if self.flagNoMoreFiles:
397 if self.flagNoMoreFiles:
358 self.dataOut.flagNoData = True
398 self.dataOut.flagNoData = True
359 self.dataOut.error = 'No More files to read'
399 self.dataOut.error = 'No More files to read'
400 return
360
401
361 if not self.readNextBlock():
402 if not self.readNextBlock():
362 self.dataOut.flagNoData = True
403 self.dataOut.flagNoData = True
363 return 0
404 self.dataOut.error = 'Time for wait new file reach!!!'
364
405
365 self.set_output()
406 self.set_output()
366
407
367 return 1
408 return 1
368 No newline at end of file
409
Show file before | Show file after | Hide comments
@@ -1,1828 +1,1833
1 '''
1 '''
2 Created on Jul 2, 2014
2 Created on Jul 2, 2014
3
3
4 @author: roj-idl71
4 @author: roj-idl71
5 '''
5 '''
6 import os
6 import os
7 import sys
7 import sys
8 import glob
8 import glob
9 import time
9 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import inspect
12 import inspect
13 import time
13 import time
14 import datetime
14 import datetime
15 import traceback
15 import traceback
16 import zmq
16 import zmq
17
17
18 try:
18 try:
19 from gevent import sleep
19 from gevent import sleep
20 except:
20 except:
21 from time import sleep
21 from time import sleep
22
22
23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 from schainpy.utils import log
25 from schainpy.utils import log
26 import schainpy.admin
26 import schainpy.admin
27
27
28 LOCALTIME = True
28 LOCALTIME = True
29
29
30
30
31 def isNumber(cad):
31 def isNumber(cad):
32 """
32 """
33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34
34
35 Excepciones:
35 Excepciones:
36 Si un determinado string no puede ser convertido a numero
36 Si un determinado string no puede ser convertido a numero
37 Input:
37 Input:
38 str, string al cual se le analiza para determinar si convertible a un numero o no
38 str, string al cual se le analiza para determinar si convertible a un numero o no
39
39
40 Return:
40 Return:
41 True : si el string es uno numerico
41 True : si el string es uno numerico
42 False : no es un string numerico
42 False : no es un string numerico
43 """
43 """
44 try:
44 try:
45 float(cad)
45 float(cad)
46 return True
46 return True
47 except:
47 except:
48 return False
48 return False
49
49
50
50
51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
52 """
52 """
53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
54
54
55 Inputs:
55 Inputs:
56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
57
57
58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
59 segundos contados desde 01/01/1970.
59 segundos contados desde 01/01/1970.
60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
61 segundos contados desde 01/01/1970.
61 segundos contados desde 01/01/1970.
62
62
63 Return:
63 Return:
64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
65 fecha especificado, de lo contrario retorna False.
65 fecha especificado, de lo contrario retorna False.
66
66
67 Excepciones:
67 Excepciones:
68 Si el archivo no existe o no puede ser abierto
68 Si el archivo no existe o no puede ser abierto
69 Si la cabecera no puede ser leida.
69 Si la cabecera no puede ser leida.
70
70
71 """
71 """
72 basicHeaderObj = BasicHeader(LOCALTIME)
72 basicHeaderObj = BasicHeader(LOCALTIME)
73
73
74 try:
74 try:
75 fp = open(filename, 'rb')
75 fp = open(filename, 'rb')
76 except IOError:
76 except IOError:
77 print("The file %s can't be opened" % (filename))
77 print("The file %s can't be opened" % (filename))
78 return 0
78 return 0
79
79
80 sts = basicHeaderObj.read(fp)
80 sts = basicHeaderObj.read(fp)
81 fp.close()
81 fp.close()
82
82
83 if not(sts):
83 if not(sts):
84 print("Skipping the file %s because it has not a valid header" % (filename))
84 print("Skipping the file %s because it has not a valid header" % (filename))
85 return 0
85 return 0
86
86
87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
88 return 0
88 return 0
89
89
90 return 1
90 return 1
91
91
92
92
93 def isTimeInRange(thisTime, startTime, endTime):
93 def isTimeInRange(thisTime, startTime, endTime):
94 if endTime >= startTime:
94 if endTime >= startTime:
95 if (thisTime < startTime) or (thisTime > endTime):
95 if (thisTime < startTime) or (thisTime > endTime):
96 return 0
96 return 0
97 return 1
97 return 1
98 else:
98 else:
99 if (thisTime < startTime) and (thisTime > endTime):
99 if (thisTime < startTime) and (thisTime > endTime):
100 return 0
100 return 0
101 return 1
101 return 1
102
102
103
103
104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
105 """
105 """
106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
107
107
108 Inputs:
108 Inputs:
109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
110
110
111 startDate : fecha inicial del rango seleccionado en formato datetime.date
111 startDate : fecha inicial del rango seleccionado en formato datetime.date
112
112
113 endDate : fecha final del rango seleccionado en formato datetime.date
113 endDate : fecha final del rango seleccionado en formato datetime.date
114
114
115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
116
116
117 endTime : tiempo final del rango seleccionado en formato datetime.time
117 endTime : tiempo final del rango seleccionado en formato datetime.time
118
118
119 Return:
119 Return:
120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
121 fecha especificado, de lo contrario retorna False.
121 fecha especificado, de lo contrario retorna False.
122
122
123 Excepciones:
123 Excepciones:
124 Si el archivo no existe o no puede ser abierto
124 Si el archivo no existe o no puede ser abierto
125 Si la cabecera no puede ser leida.
125 Si la cabecera no puede ser leida.
126
126
127 """
127 """
128
128
129 try:
129 try:
130 fp = open(filename, 'rb')
130 fp = open(filename, 'rb')
131 except IOError:
131 except IOError:
132 print("The file %s can't be opened" % (filename))
132 print("The file %s can't be opened" % (filename))
133 return None
133 return None
134
134
135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
136 systemHeaderObj = SystemHeader()
136 systemHeaderObj = SystemHeader()
137 radarControllerHeaderObj = RadarControllerHeader()
137 radarControllerHeaderObj = RadarControllerHeader()
138 processingHeaderObj = ProcessingHeader()
138 processingHeaderObj = ProcessingHeader()
139
139
140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
141
141
142 sts = firstBasicHeaderObj.read(fp)
142 sts = firstBasicHeaderObj.read(fp)
143
143
144 if not(sts):
144 if not(sts):
145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
146 return None
146 return None
147
147
148 if not systemHeaderObj.read(fp):
148 if not systemHeaderObj.read(fp):
149 return None
149 return None
150
150
151 if not radarControllerHeaderObj.read(fp):
151 if not radarControllerHeaderObj.read(fp):
152 return None
152 return None
153
153
154 if not processingHeaderObj.read(fp):
154 if not processingHeaderObj.read(fp):
155 return None
155 return None
156
156
157 filesize = os.path.getsize(filename)
157 filesize = os.path.getsize(filename)
158
158
159 offset = processingHeaderObj.blockSize + 24 # header size
159 offset = processingHeaderObj.blockSize + 24 # header size
160
160
161 if filesize <= offset:
161 if filesize <= offset:
162 print("[Reading] %s: This file has not enough data" % filename)
162 print("[Reading] %s: This file has not enough data" % filename)
163 return None
163 return None
164
164
165 fp.seek(-offset, 2)
165 fp.seek(-offset, 2)
166
166
167 sts = lastBasicHeaderObj.read(fp)
167 sts = lastBasicHeaderObj.read(fp)
168
168
169 fp.close()
169 fp.close()
170
170
171 thisDatetime = lastBasicHeaderObj.datatime
171 thisDatetime = lastBasicHeaderObj.datatime
172 thisTime_last_block = thisDatetime.time()
172 thisTime_last_block = thisDatetime.time()
173
173
174 thisDatetime = firstBasicHeaderObj.datatime
174 thisDatetime = firstBasicHeaderObj.datatime
175 thisDate = thisDatetime.date()
175 thisDate = thisDatetime.date()
176 thisTime_first_block = thisDatetime.time()
176 thisTime_first_block = thisDatetime.time()
177
177
178 # General case
178 # General case
179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
180 #-----------o----------------------------o-----------
180 #-----------o----------------------------o-----------
181 # startTime endTime
181 # startTime endTime
182
182
183 if endTime >= startTime:
183 if endTime >= startTime:
184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
185 return None
185 return None
186
186
187 return thisDatetime
187 return thisDatetime
188
188
189 # If endTime < startTime then endTime belongs to the next day
189 # If endTime < startTime then endTime belongs to the next day
190
190
191 #<<<<<<<<<<<o o>>>>>>>>>>>
191 #<<<<<<<<<<<o o>>>>>>>>>>>
192 #-----------o----------------------------o-----------
192 #-----------o----------------------------o-----------
193 # endTime startTime
193 # endTime startTime
194
194
195 if (thisDate == startDate) and (thisTime_last_block < startTime):
195 if (thisDate == startDate) and (thisTime_last_block < startTime):
196 return None
196 return None
197
197
198 if (thisDate == endDate) and (thisTime_first_block > endTime):
198 if (thisDate == endDate) and (thisTime_first_block > endTime):
199 return None
199 return None
200
200
201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
202 return None
202 return None
203
203
204 return thisDatetime
204 return thisDatetime
205
205
206
206
207 def isFolderInDateRange(folder, startDate=None, endDate=None):
207 def isFolderInDateRange(folder, startDate=None, endDate=None):
208 """
208 """
209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210
210
211 Inputs:
211 Inputs:
212 folder : nombre completo del directorio.
212 folder : nombre completo del directorio.
213 Su formato deberia ser "/path_root/?YYYYDDD"
213 Su formato deberia ser "/path_root/?YYYYDDD"
214
214
215 siendo:
215 siendo:
216 YYYY : Anio (ejemplo 2015)
216 YYYY : Anio (ejemplo 2015)
217 DDD : Dia del anio (ejemplo 305)
217 DDD : Dia del anio (ejemplo 305)
218
218
219 startDate : fecha inicial del rango seleccionado en formato datetime.date
219 startDate : fecha inicial del rango seleccionado en formato datetime.date
220
220
221 endDate : fecha final del rango seleccionado en formato datetime.date
221 endDate : fecha final del rango seleccionado en formato datetime.date
222
222
223 Return:
223 Return:
224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
225 fecha especificado, de lo contrario retorna False.
225 fecha especificado, de lo contrario retorna False.
226 Excepciones:
226 Excepciones:
227 Si el directorio no tiene el formato adecuado
227 Si el directorio no tiene el formato adecuado
228 """
228 """
229
229
230 basename = os.path.basename(folder)
230 basename = os.path.basename(folder)
231
231
232 if not isRadarFolder(basename):
232 if not isRadarFolder(basename):
233 print("The folder %s has not the rigth format" % folder)
233 print("The folder %s has not the rigth format" % folder)
234 return 0
234 return 0
235
235
236 if startDate and endDate:
236 if startDate and endDate:
237 thisDate = getDateFromRadarFolder(basename)
237 thisDate = getDateFromRadarFolder(basename)
238
238
239 if thisDate < startDate:
239 if thisDate < startDate:
240 return 0
240 return 0
241
241
242 if thisDate > endDate:
242 if thisDate > endDate:
243 return 0
243 return 0
244
244
245 return 1
245 return 1
246
246
247
247
248 def isFileInDateRange(filename, startDate=None, endDate=None):
248 def isFileInDateRange(filename, startDate=None, endDate=None):
249 """
249 """
250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
251
251
252 Inputs:
252 Inputs:
253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
254
254
255 Su formato deberia ser "?YYYYDDDsss"
255 Su formato deberia ser "?YYYYDDDsss"
256
256
257 siendo:
257 siendo:
258 YYYY : Anio (ejemplo 2015)
258 YYYY : Anio (ejemplo 2015)
259 DDD : Dia del anio (ejemplo 305)
259 DDD : Dia del anio (ejemplo 305)
260 sss : set
260 sss : set
261
261
262 startDate : fecha inicial del rango seleccionado en formato datetime.date
262 startDate : fecha inicial del rango seleccionado en formato datetime.date
263
263
264 endDate : fecha final del rango seleccionado en formato datetime.date
264 endDate : fecha final del rango seleccionado en formato datetime.date
265
265
266 Return:
266 Return:
267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
268 fecha especificado, de lo contrario retorna False.
268 fecha especificado, de lo contrario retorna False.
269 Excepciones:
269 Excepciones:
270 Si el archivo no tiene el formato adecuado
270 Si el archivo no tiene el formato adecuado
271 """
271 """
272
272
273 basename = os.path.basename(filename)
273 basename = os.path.basename(filename)
274
274
275 if not isRadarFile(basename):
275 if not isRadarFile(basename):
276 print("The filename %s has not the rigth format" % filename)
276 print("The filename %s has not the rigth format" % filename)
277 return 0
277 return 0
278
278
279 if startDate and endDate:
279 if startDate and endDate:
280 thisDate = getDateFromRadarFile(basename)
280 thisDate = getDateFromRadarFile(basename)
281
281
282 if thisDate < startDate:
282 if thisDate < startDate:
283 return 0
283 return 0
284
284
285 if thisDate > endDate:
285 if thisDate > endDate:
286 return 0
286 return 0
287
287
288 return 1
288 return 1
289
289
290
290
291 def getFileFromSet(path, ext, set):
291 def getFileFromSet(path, ext, set):
292 validFilelist = []
292 validFilelist = []
293 fileList = os.listdir(path)
293 fileList = os.listdir(path)
294
294
295 # 0 1234 567 89A BCDE
295 # 0 1234 567 89A BCDE
296 # H YYYY DDD SSS .ext
296 # H YYYY DDD SSS .ext
297
297
298 for thisFile in fileList:
298 for thisFile in fileList:
299 try:
299 try:
300 year = int(thisFile[1:5])
300 year = int(thisFile[1:5])
301 doy = int(thisFile[5:8])
301 doy = int(thisFile[5:8])
302 except:
302 except:
303 continue
303 continue
304
304
305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 continue
306 continue
307
307
308 validFilelist.append(thisFile)
308 validFilelist.append(thisFile)
309
309
310 myfile = fnmatch.filter(
310 myfile = fnmatch.filter(
311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
312
312
313 if len(myfile) != 0:
313 if len(myfile) != 0:
314 return myfile[0]
314 return myfile[0]
315 else:
315 else:
316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
317 print('the filename %s does not exist' % filename)
317 print('the filename %s does not exist' % filename)
318 print('...going to the last file: ')
318 print('...going to the last file: ')
319
319
320 if validFilelist:
320 if validFilelist:
321 validFilelist = sorted(validFilelist, key=str.lower)
321 validFilelist = sorted(validFilelist, key=str.lower)
322 return validFilelist[-1]
322 return validFilelist[-1]
323
323
324 return None
324 return None
325
325
326
326
327 def getlastFileFromPath(path, ext):
327 def getlastFileFromPath(path, ext):
328 """
328 """
329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
331
331
332 Input:
332 Input:
333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
334 ext : extension de los files contenidos en una carpeta
334 ext : extension de los files contenidos en una carpeta
335
335
336 Return:
336 Return:
337 El ultimo file de una determinada carpeta, no se considera el path.
337 El ultimo file de una determinada carpeta, no se considera el path.
338 """
338 """
339 validFilelist = []
339 validFilelist = []
340 fileList = os.listdir(path)
340 fileList = os.listdir(path)
341
341
342 # 0 1234 567 89A BCDE
342 # 0 1234 567 89A BCDE
343 # H YYYY DDD SSS .ext
343 # H YYYY DDD SSS .ext
344
344
345 for thisFile in fileList:
345 for thisFile in fileList:
346
346
347 year = thisFile[1:5]
347 year = thisFile[1:5]
348 if not isNumber(year):
348 if not isNumber(year):
349 continue
349 continue
350
350
351 doy = thisFile[5:8]
351 doy = thisFile[5:8]
352 if not isNumber(doy):
352 if not isNumber(doy):
353 continue
353 continue
354
354
355 year = int(year)
355 year = int(year)
356 doy = int(doy)
356 doy = int(doy)
357
357
358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
359 continue
359 continue
360
360
361 validFilelist.append(thisFile)
361 validFilelist.append(thisFile)
362
362
363 if validFilelist:
363 if validFilelist:
364 validFilelist = sorted(validFilelist, key=str.lower)
364 validFilelist = sorted(validFilelist, key=str.lower)
365 return validFilelist[-1]
365 return validFilelist[-1]
366
366
367 return None
367 return None
368
368
369
369
370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
371 """
371 """
372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
374 el path exacto de un determinado file.
374 el path exacto de un determinado file.
375
375
376 Example :
376 Example :
377 nombre correcto del file es .../.../D2009307/P2009307367.ext
377 nombre correcto del file es .../.../D2009307/P2009307367.ext
378
378
379 Entonces la funcion prueba con las siguientes combinaciones
379 Entonces la funcion prueba con las siguientes combinaciones
380 .../.../y2009307367.ext
380 .../.../y2009307367.ext
381 .../.../Y2009307367.ext
381 .../.../Y2009307367.ext
382 .../.../x2009307/y2009307367.ext
382 .../.../x2009307/y2009307367.ext
383 .../.../x2009307/Y2009307367.ext
383 .../.../x2009307/Y2009307367.ext
384 .../.../X2009307/y2009307367.ext
384 .../.../X2009307/y2009307367.ext
385 .../.../X2009307/Y2009307367.ext
385 .../.../X2009307/Y2009307367.ext
386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
387
387
388 Return:
388 Return:
389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
391 para el filename
391 para el filename
392 """
392 """
393 fullfilename = None
393 fullfilename = None
394 find_flag = False
394 find_flag = False
395 filename = None
395 filename = None
396
396
397 prefixDirList = [None, 'd', 'D']
397 prefixDirList = [None, 'd', 'D']
398 if ext.lower() == ".r": # voltage
398 if ext.lower() == ".r": # voltage
399 prefixFileList = ['d', 'D']
399 prefixFileList = ['d', 'D']
400 elif ext.lower() == ".pdata": # spectra
400 elif ext.lower() == ".pdata": # spectra
401 prefixFileList = ['p', 'P']
401 prefixFileList = ['p', 'P']
402 else:
402 else:
403 return None, filename
403 return None, filename
404
404
405 # barrido por las combinaciones posibles
405 # barrido por las combinaciones posibles
406 for prefixDir in prefixDirList:
406 for prefixDir in prefixDirList:
407 thispath = path
407 thispath = path
408 if prefixDir != None:
408 if prefixDir != None:
409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
410 if foldercounter == 0:
410 if foldercounter == 0:
411 thispath = os.path.join(path, "%s%04d%03d" %
411 thispath = os.path.join(path, "%s%04d%03d" %
412 (prefixDir, year, doy))
412 (prefixDir, year, doy))
413 else:
413 else:
414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
415 prefixDir, year, doy, foldercounter))
415 prefixDir, year, doy, foldercounter))
416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
417 # formo el nombre del file xYYYYDDDSSS.ext
417 # formo el nombre del file xYYYYDDDSSS.ext
418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
419 fullfilename = os.path.join(
419 fullfilename = os.path.join(
420 thispath, filename) # formo el path completo
420 thispath, filename) # formo el path completo
421
421
422 if os.path.exists(fullfilename): # verifico que exista
422 if os.path.exists(fullfilename): # verifico que exista
423 find_flag = True
423 find_flag = True
424 break
424 break
425 if find_flag:
425 if find_flag:
426 break
426 break
427
427
428 if not(find_flag):
428 if not(find_flag):
429 return None, filename
429 return None, filename
430
430
431 return fullfilename, filename
431 return fullfilename, filename
432
432
433
433
434 def isRadarFolder(folder):
434 def isRadarFolder(folder):
435 try:
435 try:
436 year = int(folder[1:5])
436 year = int(folder[1:5])
437 doy = int(folder[5:8])
437 doy = int(folder[5:8])
438 except:
438 except:
439 return 0
439 return 0
440
440
441 return 1
441 return 1
442
442
443
443
444 def isRadarFile(file):
444 def isRadarFile(file):
445 try:
445 try:
446 year = int(file[1:5])
446 year = int(file[1:5])
447 doy = int(file[5:8])
447 doy = int(file[5:8])
448 set = int(file[8:11])
448 set = int(file[8:11])
449 except:
449 except:
450 return 0
450 return 0
451
451
452 return 1
452 return 1
453
453
454
454
455 def getDateFromRadarFile(file):
455 def getDateFromRadarFile(file):
456 try:
456 try:
457 year = int(file[1:5])
457 year = int(file[1:5])
458 doy = int(file[5:8])
458 doy = int(file[5:8])
459 set = int(file[8:11])
459 set = int(file[8:11])
460 except:
460 except:
461 return None
461 return None
462
462
463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
464 return thisDate
464 return thisDate
465
465
466
466
467 def getDateFromRadarFolder(folder):
467 def getDateFromRadarFolder(folder):
468 try:
468 try:
469 year = int(folder[1:5])
469 year = int(folder[1:5])
470 doy = int(folder[5:8])
470 doy = int(folder[5:8])
471 except:
471 except:
472 return None
472 return None
473
473
474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
475 return thisDate
475 return thisDate
476
476
477
477
478 class JRODataIO:
478 class JRODataIO:
479
479
480 c = 3E8
480 c = 3E8
481
481
482 isConfig = False
482 isConfig = False
483
483
484 basicHeaderObj = None
484 basicHeaderObj = None
485
485
486 systemHeaderObj = None
486 systemHeaderObj = None
487
487
488 radarControllerHeaderObj = None
488 radarControllerHeaderObj = None
489
489
490 processingHeaderObj = None
490 processingHeaderObj = None
491
491
492 dtype = None
492 dtype = None
493
493
494 pathList = []
494 pathList = []
495
495
496 filenameList = []
496 filenameList = []
497
497
498 filename = None
498 filename = None
499
499
500 ext = None
500 ext = None
501
501
502 flagIsNewFile = 1
502 flagIsNewFile = 1
503
503
504 flagDiscontinuousBlock = 0
504 flagDiscontinuousBlock = 0
505
505
506 flagIsNewBlock = 0
506 flagIsNewBlock = 0
507
507
508 fp = None
508 fp = None
509
509
510 firstHeaderSize = 0
510 firstHeaderSize = 0
511
511
512 basicHeaderSize = 24
512 basicHeaderSize = 24
513
513
514 versionFile = 1103
514 versionFile = 1103
515
515
516 fileSize = None
516 fileSize = None
517
517
518 # ippSeconds = None
518 # ippSeconds = None
519
519
520 fileSizeByHeader = None
520 fileSizeByHeader = None
521
521
522 fileIndex = None
522 fileIndex = None
523
523
524 profileIndex = None
524 profileIndex = None
525
525
526 blockIndex = None
526 blockIndex = None
527
527
528 nTotalBlocks = None
528 nTotalBlocks = None
529
529
530 maxTimeStep = 30
530 maxTimeStep = 30
531
531
532 lastUTTime = None
532 lastUTTime = None
533
533
534 datablock = None
534 datablock = None
535
535
536 dataOut = None
536 dataOut = None
537
537
538 blocksize = None
538 blocksize = None
539
539
540 getByBlock = False
540 getByBlock = False
541
541
542 def __init__(self):
542 def __init__(self):
543
543
544 raise NotImplementedError
544 raise NotImplementedError
545
545
546 def run(self):
546 def run(self):
547
547
548 raise NotImplementedError
548 raise NotImplementedError
549
549
550 def getDtypeWidth(self):
550 def getDtypeWidth(self):
551
551
552 dtype_index = get_dtype_index(self.dtype)
552 dtype_index = get_dtype_index(self.dtype)
553 dtype_width = get_dtype_width(dtype_index)
553 dtype_width = get_dtype_width(dtype_index)
554
554
555 return dtype_width
555 return dtype_width
556
556
557 def getAllowedArgs(self):
557 def getAllowedArgs(self):
558 if hasattr(self, '__attrs__'):
558 if hasattr(self, '__attrs__'):
559 return self.__attrs__
559 return self.__attrs__
560 else:
560 else:
561 return inspect.getargspec(self.run).args
561 return inspect.getargspec(self.run).args
562
562
563
563
564 class JRODataReader(JRODataIO):
564 class JRODataReader(JRODataIO):
565
565
566 online = 0
566 online = 0
567
567
568 realtime = 0
568 realtime = 0
569
569
570 nReadBlocks = 0
570 nReadBlocks = 0
571
571
572 delay = 10 # number of seconds waiting a new file
572 delay = 10 # number of seconds waiting a new file
573
573
574 nTries = 3 # quantity tries
574 nTries = 3 # quantity tries
575
575
576 nFiles = 3 # number of files for searching
576 nFiles = 3 # number of files for searching
577
577
578 path = None
578 path = None
579
579
580 foldercounter = 0
580 foldercounter = 0
581
581
582 flagNoMoreFiles = 0
582 flagNoMoreFiles = 0
583
583
584 datetimeList = []
584 datetimeList = []
585
585
586 __isFirstTimeOnline = 1
586 __isFirstTimeOnline = 1
587
587
588 __printInfo = True
588 __printInfo = True
589
589
590 profileIndex = None
590 profileIndex = None
591
591
592 nTxs = 1
592 nTxs = 1
593
593
594 txIndex = None
594 txIndex = None
595
595
596 # Added--------------------
596 # Added--------------------
597
597
598 selBlocksize = None
598 selBlocksize = None
599
599
600 selBlocktime = None
600 selBlocktime = None
601
601
602 def __init__(self):
602 def __init__(self):
603 """
603 """
604 This class is used to find data files
604 This class is used to find data files
605
605
606 Example:
606 Example:
607 reader = JRODataReader()
607 reader = JRODataReader()
608 fileList = reader.findDataFiles()
608 fileList = reader.findDataFiles()
609
609
610 """
610 """
611 pass
611 pass
612
612
613 def createObjByDefault(self):
613 def createObjByDefault(self):
614 """
614 """
615
615
616 """
616 """
617 raise NotImplementedError
617 raise NotImplementedError
618
618
619 def getBlockDimension(self):
619 def getBlockDimension(self):
620
620
621 raise NotImplementedError
621 raise NotImplementedError
622
622
623 def searchFilesOffLine(self,
623 def searchFilesOffLine(self,
624 path,
624 path,
625 startDate=None,
625 startDate=None,
626 endDate=None,
626 endDate=None,
627 startTime=datetime.time(0, 0, 0),
627 startTime=datetime.time(0, 0, 0),
628 endTime=datetime.time(23, 59, 59),
628 endTime=datetime.time(23, 59, 59),
629 set=None,
629 set=None,
630 expLabel='',
630 expLabel='',
631 ext='.r',
631 ext='.r',
632 cursor=None,
632 cursor=None,
633 skip=None,
633 skip=None,
634 walk=True):
634 walk=True):
635
635
636 self.filenameList = []
636 self.filenameList = []
637 self.datetimeList = []
637 self.datetimeList = []
638
638
639 pathList = []
639 pathList = []
640
640
641 dateList, pathList = self.findDatafiles(
641 dateList, pathList = self.findDatafiles(
642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
643
643
644 if dateList == []:
644 if dateList == []:
645 return [], []
645 return [], []
646
646
647 if len(dateList) > 1:
647 if len(dateList) > 1:
648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
649 else:
649 else:
650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
651
651
652 filenameList = []
652 filenameList = []
653 datetimeList = []
653 datetimeList = []
654
654
655 for thisPath in pathList:
655 for thisPath in pathList:
656
656
657 fileList = glob.glob1(thisPath, "*%s" % ext)
657 fileList = glob.glob1(thisPath, "*%s" % ext)
658 fileList.sort()
658 fileList.sort()
659
659
660 for file in fileList:
660 for file in fileList:
661
661
662 filename = os.path.join(thisPath, file)
662 filename = os.path.join(thisPath, file)
663
663
664 if not isFileInDateRange(filename, startDate, endDate):
664 if not isFileInDateRange(filename, startDate, endDate):
665 continue
665 continue
666
666
667 thisDatetime = isFileInTimeRange(
667 thisDatetime = isFileInTimeRange(
668 filename, startDate, endDate, startTime, endTime)
668 filename, startDate, endDate, startTime, endTime)
669
669
670 if not(thisDatetime):
670 if not(thisDatetime):
671 continue
671 continue
672
672
673 filenameList.append(filename)
673 filenameList.append(filename)
674 datetimeList.append(thisDatetime)
674 datetimeList.append(thisDatetime)
675
675
676 if cursor is not None and skip is not None:
676 if cursor is not None and skip is not None:
677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
679
679
680 if not(filenameList):
680 if not(filenameList):
681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
682 return [], []
682 return [], []
683
683
684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
685
685
686 # for i in range(len(filenameList)):
686 # for i in range(len(filenameList)):
687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
688
688
689 self.filenameList = filenameList
689 self.filenameList = filenameList
690 self.datetimeList = datetimeList
690 self.datetimeList = datetimeList
691
691
692 return pathList, filenameList
692 return pathList, filenameList
693
693
694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
695 """
695 """
696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
697 devuelve el archivo encontrado ademas de otros datos.
697 devuelve el archivo encontrado ademas de otros datos.
698
698
699 Input:
699 Input:
700 path : carpeta donde estan contenidos los files que contiene data
700 path : carpeta donde estan contenidos los files que contiene data
701
701
702 expLabel : Nombre del subexperimento (subfolder)
702 expLabel : Nombre del subexperimento (subfolder)
703
703
704 ext : extension de los files
704 ext : extension de los files
705
705
706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
707
707
708 Return:
708 Return:
709 directory : eL directorio donde esta el file encontrado
709 directory : eL directorio donde esta el file encontrado
710 filename : el ultimo file de una determinada carpeta
710 filename : el ultimo file de una determinada carpeta
711 year : el anho
711 year : el anho
712 doy : el numero de dia del anho
712 doy : el numero de dia del anho
713 set : el set del archivo
713 set : el set del archivo
714
714
715
715
716 """
716 """
717 if not os.path.isdir(path):
717 if not os.path.isdir(path):
718 return None, None, None, None, None, None
718 return None, None, None, None, None, None
719
719
720 dirList = []
720 dirList = []
721
721
722 if not walk:
722 if not walk:
723 fullpath = path
723 fullpath = path
724 foldercounter = 0
724 foldercounter = 0
725 else:
725 else:
726 # Filtra solo los directorios
726 # Filtra solo los directorios
727 for thisPath in os.listdir(path):
727 for thisPath in os.listdir(path):
728 if not os.path.isdir(os.path.join(path, thisPath)):
728 if not os.path.isdir(os.path.join(path, thisPath)):
729 continue
729 continue
730 if not isRadarFolder(thisPath):
730 if not isRadarFolder(thisPath):
731 continue
731 continue
732
732
733 dirList.append(thisPath)
733 dirList.append(thisPath)
734
734
735 if not(dirList):
735 if not(dirList):
736 return None, None, None, None, None, None
736 return None, None, None, None, None, None
737
737
738 dirList = sorted(dirList, key=str.lower)
738 dirList = sorted(dirList, key=str.lower)
739
739
740 doypath = dirList[-1]
740 doypath = dirList[-1]
741 foldercounter = int(doypath.split('_')[1]) if len(
741 foldercounter = int(doypath.split('_')[1]) if len(
742 doypath.split('_')) > 1 else 0
742 doypath.split('_')) > 1 else 0
743 fullpath = os.path.join(path, doypath, expLabel)
743 fullpath = os.path.join(path, doypath, expLabel)
744
744
745 print("[Reading] %s folder was found: " % (fullpath))
745 print("[Reading] %s folder was found: " % (fullpath))
746
746
747 if set == None:
747 if set == None:
748 filename = getlastFileFromPath(fullpath, ext)
748 filename = getlastFileFromPath(fullpath, ext)
749 else:
749 else:
750 filename = getFileFromSet(fullpath, ext, set)
750 filename = getFileFromSet(fullpath, ext, set)
751
751
752 if not(filename):
752 if not(filename):
753 return None, None, None, None, None, None
753 return None, None, None, None, None, None
754
754
755 print("[Reading] %s file was found" % (filename))
755 print("[Reading] %s file was found" % (filename))
756
756
757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
758 return None, None, None, None, None, None
758 return None, None, None, None, None, None
759
759
760 year = int(filename[1:5])
760 year = int(filename[1:5])
761 doy = int(filename[5:8])
761 doy = int(filename[5:8])
762 set = int(filename[8:11])
762 set = int(filename[8:11])
763
763
764 return fullpath, foldercounter, filename, year, doy, set
764 return fullpath, foldercounter, filename, year, doy, set
765
765
766 def __setNextFileOffline(self):
766 def __setNextFileOffline(self):
767
767
768 idFile = self.fileIndex
768 idFile = self.fileIndex
769
769
770 while (True):
770 while (True):
771 idFile += 1
771 idFile += 1
772 if not(idFile < len(self.filenameList)):
772 if not(idFile < len(self.filenameList)):
773 self.flagNoMoreFiles = 1
773 self.flagNoMoreFiles = 1
774 # print "[Reading] No more Files"
774 # print "[Reading] No more Files"
775 return 0
775 return 0
776
776
777 filename = self.filenameList[idFile]
777 filename = self.filenameList[idFile]
778
778
779 if not(self.__verifyFile(filename)):
779 if not(self.__verifyFile(filename)):
780 continue
780 continue
781
781
782 fileSize = os.path.getsize(filename)
782 fileSize = os.path.getsize(filename)
783 fp = open(filename, 'rb')
783 fp = open(filename, 'rb')
784 break
784 break
785
785
786 self.flagIsNewFile = 1
786 self.flagIsNewFile = 1
787 self.fileIndex = idFile
787 self.fileIndex = idFile
788 self.filename = filename
788 self.filename = filename
789 self.fileSize = fileSize
789 self.fileSize = fileSize
790 self.fp = fp
790 self.fp = fp
791
791
792 # print "[Reading] Setting the file: %s"%self.filename
792 # print "[Reading] Setting the file: %s"%self.filename
793
793
794 return 1
794 return 1
795
795
796 def __setNextFileOnline(self):
796 def __setNextFileOnline(self):
797 """
797 """
798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
800 siguientes.
800 siguientes.
801
801
802 Affected:
802 Affected:
803 self.flagIsNewFile
803 self.flagIsNewFile
804 self.filename
804 self.filename
805 self.fileSize
805 self.fileSize
806 self.fp
806 self.fp
807 self.set
807 self.set
808 self.flagNoMoreFiles
808 self.flagNoMoreFiles
809
809
810 Return:
810 Return:
811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
812 1 : si el file fue abierto con exito y esta listo a ser leido
812 1 : si el file fue abierto con exito y esta listo a ser leido
813
813
814 Excepciones:
814 Excepciones:
815 Si un determinado file no puede ser abierto
815 Si un determinado file no puede ser abierto
816 """
816 """
817 nFiles = 0
817 nFiles = 0
818 fileOk_flag = False
818 fileOk_flag = False
819 firstTime_flag = True
819 firstTime_flag = True
820
820
821 self.set += 1
821 self.set += 1
822
822
823 if self.set > 999:
823 if self.set > 999:
824 self.set = 0
824 self.set = 0
825 self.foldercounter += 1
825 self.foldercounter += 1
826
826
827 # busca el 1er file disponible
827 # busca el 1er file disponible
828 fullfilename, filename = checkForRealPath(
828 fullfilename, filename = checkForRealPath(
829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
830 if fullfilename:
830 if fullfilename:
831 if self.__verifyFile(fullfilename, False):
831 if self.__verifyFile(fullfilename, False):
832 fileOk_flag = True
832 fileOk_flag = True
833
833
834 # si no encuentra un file entonces espera y vuelve a buscar
834 # si no encuentra un file entonces espera y vuelve a buscar
835 if not(fileOk_flag):
835 if not(fileOk_flag):
836 # busco en los siguientes self.nFiles+1 files posibles
836 # busco en los siguientes self.nFiles+1 files posibles
837 for nFiles in range(self.nFiles + 1):
837 for nFiles in range(self.nFiles + 1):
838
838
839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
840 tries = self.nTries
840 tries = self.nTries
841 else:
841 else:
842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
843
843
844 for nTries in range(tries):
844 for nTries in range(tries):
845 if firstTime_flag:
845 if firstTime_flag:
846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
847 sleep(self.delay)
847 sleep(self.delay)
848 else:
848 else:
849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
850
850
851 fullfilename, filename = checkForRealPath(
851 fullfilename, filename = checkForRealPath(
852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
853 if fullfilename:
853 if fullfilename:
854 if self.__verifyFile(fullfilename):
854 if self.__verifyFile(fullfilename):
855 fileOk_flag = True
855 fileOk_flag = True
856 break
856 break
857
857
858 if fileOk_flag:
858 if fileOk_flag:
859 break
859 break
860
860
861 firstTime_flag = False
861 firstTime_flag = False
862
862
863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
864 self.set += 1
864 self.set += 1
865
865
866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
867 if nFiles == (self.nFiles - 1):
867 if nFiles == (self.nFiles - 1):
868 self.set = 0
868 self.set = 0
869 self.doy += 1
869 self.doy += 1
870 self.foldercounter = 0
870 self.foldercounter = 0
871
871
872 if fileOk_flag:
872 if fileOk_flag:
873 self.fileSize = os.path.getsize(fullfilename)
873 self.fileSize = os.path.getsize(fullfilename)
874 self.filename = fullfilename
874 self.filename = fullfilename
875 self.flagIsNewFile = 1
875 self.flagIsNewFile = 1
876 if self.fp != None:
876 if self.fp != None:
877 self.fp.close()
877 self.fp.close()
878 self.fp = open(fullfilename, 'rb')
878 self.fp = open(fullfilename, 'rb')
879 self.flagNoMoreFiles = 0
879 self.flagNoMoreFiles = 0
880 # print '[Reading] Setting the file: %s' % fullfilename
880 # print '[Reading] Setting the file: %s' % fullfilename
881 else:
881 else:
882 self.fileSize = 0
882 self.fileSize = 0
883 self.filename = None
883 self.filename = None
884 self.flagIsNewFile = 0
884 self.flagIsNewFile = 0
885 self.fp = None
885 self.fp = None
886 self.flagNoMoreFiles = 1
886 self.flagNoMoreFiles = 1
887 # print '[Reading] No more files to read'
887 # print '[Reading] No more files to read'
888
888
889 return fileOk_flag
889 return fileOk_flag
890
890
891 def setNextFile(self):
891 def setNextFile(self):
892 if self.fp != None:
892 if self.fp != None:
893 self.fp.close()
893 self.fp.close()
894
894
895 if self.online:
895 if self.online:
896 newFile = self.__setNextFileOnline()
896 newFile = self.__setNextFileOnline()
897 else:
897 else:
898 newFile = self.__setNextFileOffline()
898 newFile = self.__setNextFileOffline()
899
899
900 if not(newFile):
900 if not(newFile):
901 self.dataOut.error = 'No more files to read'
901 self.dataOut.error = 'No more files to read'
902 return 0
902 return 0
903
903
904 if self.verbose:
904 if self.verbose:
905 print('[Reading] Setting the file: %s' % self.filename)
905 print('[Reading] Setting the file: %s' % self.filename)
906
906
907 self.__readFirstHeader()
907 self.__readFirstHeader()
908 self.nReadBlocks = 0
908 self.nReadBlocks = 0
909 return 1
909 return 1
910
910
911 def __waitNewBlock(self):
911 def __waitNewBlock(self):
912 """
912 """
913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914
914
915 Si el modo de lectura es OffLine siempre retorn 0
915 Si el modo de lectura es OffLine siempre retorn 0
916 """
916 """
917 if not self.online:
917 if not self.online:
918 return 0
918 return 0
919
919
920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 return 0
921 return 0
922
922
923 currentPointer = self.fp.tell()
923 currentPointer = self.fp.tell()
924
924
925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926
926
927 for nTries in range(self.nTries):
927 for nTries in range(self.nTries):
928
928
929 self.fp.close()
929 self.fp.close()
930 self.fp = open(self.filename, 'rb')
930 self.fp = open(self.filename, 'rb')
931 self.fp.seek(currentPointer)
931 self.fp.seek(currentPointer)
932
932
933 self.fileSize = os.path.getsize(self.filename)
933 self.fileSize = os.path.getsize(self.filename)
934 currentSize = self.fileSize - currentPointer
934 currentSize = self.fileSize - currentPointer
935
935
936 if (currentSize >= neededSize):
936 if (currentSize >= neededSize):
937 self.basicHeaderObj.read(self.fp)
937 self.basicHeaderObj.read(self.fp)
938 return 1
938 return 1
939
939
940 if self.fileSize == self.fileSizeByHeader:
940 if self.fileSize == self.fileSizeByHeader:
941 # self.flagEoF = True
941 # self.flagEoF = True
942 return 0
942 return 0
943
943
944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 sleep(self.delay)
945 sleep(self.delay)
946
946
947 return 0
947 return 0
948
948
949 def waitDataBlock(self, pointer_location):
949 def waitDataBlock(self, pointer_location, blocksize=None):
950
950
951 currentPointer = pointer_location
951 currentPointer = pointer_location
952
952 if blocksize is None:
953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 else:
955 neededSize = blocksize
954
956
955 for nTries in range(self.nTries):
957 for nTries in range(self.nTries):
956 self.fp.close()
958 self.fp.close()
957 self.fp = open(self.filename, 'rb')
959 self.fp = open(self.filename, 'rb')
958 self.fp.seek(currentPointer)
960 self.fp.seek(currentPointer)
959
961
960 self.fileSize = os.path.getsize(self.filename)
962 self.fileSize = os.path.getsize(self.filename)
961 currentSize = self.fileSize - currentPointer
963 currentSize = self.fileSize - currentPointer
962
964
963 if (currentSize >= neededSize):
965 if (currentSize >= neededSize):
964 return 1
966 return 1
965
967
966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
968 log.warning(
969 "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
970 self.name
971 )
967 sleep(self.delay)
972 sleep(self.delay)
968
973
969 return 0
974 return 0
970
975
971 def __jumpToLastBlock(self):
976 def __jumpToLastBlock(self):
972
977
973 if not(self.__isFirstTimeOnline):
978 if not(self.__isFirstTimeOnline):
974 return
979 return
975
980
976 csize = self.fileSize - self.fp.tell()
981 csize = self.fileSize - self.fp.tell()
977 blocksize = self.processingHeaderObj.blockSize
982 blocksize = self.processingHeaderObj.blockSize
978
983
979 # salta el primer bloque de datos
984 # salta el primer bloque de datos
980 if csize > self.processingHeaderObj.blockSize:
985 if csize > self.processingHeaderObj.blockSize:
981 self.fp.seek(self.fp.tell() + blocksize)
986 self.fp.seek(self.fp.tell() + blocksize)
982 else:
987 else:
983 return
988 return
984
989
985 csize = self.fileSize - self.fp.tell()
990 csize = self.fileSize - self.fp.tell()
986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
991 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
987 while True:
992 while True:
988
993
989 if self.fp.tell() < self.fileSize:
994 if self.fp.tell() < self.fileSize:
990 self.fp.seek(self.fp.tell() + neededsize)
995 self.fp.seek(self.fp.tell() + neededsize)
991 else:
996 else:
992 self.fp.seek(self.fp.tell() - neededsize)
997 self.fp.seek(self.fp.tell() - neededsize)
993 break
998 break
994
999
995 # csize = self.fileSize - self.fp.tell()
1000 # csize = self.fileSize - self.fp.tell()
996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
997 # factor = int(csize/neededsize)
1002 # factor = int(csize/neededsize)
998 # if factor > 0:
1003 # if factor > 0:
999 # self.fp.seek(self.fp.tell() + factor*neededsize)
1004 # self.fp.seek(self.fp.tell() + factor*neededsize)
1000
1005
1001 self.flagIsNewFile = 0
1006 self.flagIsNewFile = 0
1002 self.__isFirstTimeOnline = 0
1007 self.__isFirstTimeOnline = 0
1003
1008
1004 def __setNewBlock(self):
1009 def __setNewBlock(self):
1005 # if self.server is None:
1010 # if self.server is None:
1006 if self.fp == None:
1011 if self.fp == None:
1007 return 0
1012 return 0
1008
1013
1009 # if self.online:
1014 # if self.online:
1010 # self.__jumpToLastBlock()
1015 # self.__jumpToLastBlock()
1011
1016
1012 if self.flagIsNewFile:
1017 if self.flagIsNewFile:
1013 self.lastUTTime = self.basicHeaderObj.utc
1018 self.lastUTTime = self.basicHeaderObj.utc
1014 return 1
1019 return 1
1015
1020
1016 if self.realtime:
1021 if self.realtime:
1017 self.flagDiscontinuousBlock = 1
1022 self.flagDiscontinuousBlock = 1
1018 if not(self.setNextFile()):
1023 if not(self.setNextFile()):
1019 return 0
1024 return 0
1020 else:
1025 else:
1021 return 1
1026 return 1
1022 # if self.server is None:
1027 # if self.server is None:
1023 currentSize = self.fileSize - self.fp.tell()
1028 currentSize = self.fileSize - self.fp.tell()
1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1029 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1025 if (currentSize >= neededSize):
1030 if (currentSize >= neededSize):
1026 self.basicHeaderObj.read(self.fp)
1031 self.basicHeaderObj.read(self.fp)
1027 self.lastUTTime = self.basicHeaderObj.utc
1032 self.lastUTTime = self.basicHeaderObj.utc
1028 return 1
1033 return 1
1029 # else:
1034 # else:
1030 # self.basicHeaderObj.read(self.zHeader)
1035 # self.basicHeaderObj.read(self.zHeader)
1031 # self.lastUTTime = self.basicHeaderObj.utc
1036 # self.lastUTTime = self.basicHeaderObj.utc
1032 # return 1
1037 # return 1
1033 if self.__waitNewBlock():
1038 if self.__waitNewBlock():
1034 self.lastUTTime = self.basicHeaderObj.utc
1039 self.lastUTTime = self.basicHeaderObj.utc
1035 return 1
1040 return 1
1036 # if self.server is None:
1041 # if self.server is None:
1037 if not(self.setNextFile()):
1042 if not(self.setNextFile()):
1038 return 0
1043 return 0
1039
1044
1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1045 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1041 self.lastUTTime = self.basicHeaderObj.utc
1046 self.lastUTTime = self.basicHeaderObj.utc
1042
1047
1043 self.flagDiscontinuousBlock = 0
1048 self.flagDiscontinuousBlock = 0
1044
1049
1045 if deltaTime > self.maxTimeStep:
1050 if deltaTime > self.maxTimeStep:
1046 self.flagDiscontinuousBlock = 1
1051 self.flagDiscontinuousBlock = 1
1047
1052
1048 return 1
1053 return 1
1049
1054
1050 def readNextBlock(self):
1055 def readNextBlock(self):
1051
1056
1052 # Skip block out of startTime and endTime
1057 # Skip block out of startTime and endTime
1053 while True:
1058 while True:
1054 if not(self.__setNewBlock()):
1059 if not(self.__setNewBlock()):
1055 self.dataOut.error = 'No more files to read'
1060 self.dataOut.error = 'No more files to read'
1056 return 0
1061 return 0
1057
1062
1058 if not(self.readBlock()):
1063 if not(self.readBlock()):
1059 return 0
1064 return 0
1060
1065
1061 self.getBasicHeader()
1066 self.getBasicHeader()
1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1067 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1063 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1068 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1064 self.processingHeaderObj.dataBlocksPerFile,
1069 self.processingHeaderObj.dataBlocksPerFile,
1065 self.dataOut.datatime.ctime()))
1070 self.dataOut.datatime.ctime()))
1066 continue
1071 continue
1067
1072
1068 break
1073 break
1069
1074
1070 if self.verbose:
1075 if self.verbose:
1071 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1076 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1072 self.processingHeaderObj.dataBlocksPerFile,
1077 self.processingHeaderObj.dataBlocksPerFile,
1073 self.dataOut.datatime.ctime()))
1078 self.dataOut.datatime.ctime()))
1074 return 1
1079 return 1
1075
1080
1076 def __readFirstHeader(self):
1081 def __readFirstHeader(self):
1077
1082
1078 self.basicHeaderObj.read(self.fp)
1083 self.basicHeaderObj.read(self.fp)
1079 self.systemHeaderObj.read(self.fp)
1084 self.systemHeaderObj.read(self.fp)
1080 self.radarControllerHeaderObj.read(self.fp)
1085 self.radarControllerHeaderObj.read(self.fp)
1081 self.processingHeaderObj.read(self.fp)
1086 self.processingHeaderObj.read(self.fp)
1082
1087
1083 self.firstHeaderSize = self.basicHeaderObj.size
1088 self.firstHeaderSize = self.basicHeaderObj.size
1084
1089
1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1090 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1091 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1087 if datatype == 0:
1092 if datatype == 0:
1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1093 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1089 elif datatype == 1:
1094 elif datatype == 1:
1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1095 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1091 elif datatype == 2:
1096 elif datatype == 2:
1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1097 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1093 elif datatype == 3:
1098 elif datatype == 3:
1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1099 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1095 elif datatype == 4:
1100 elif datatype == 4:
1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1101 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1097 elif datatype == 5:
1102 elif datatype == 5:
1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1103 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1099 else:
1104 else:
1100 raise ValueError('Data type was not defined')
1105 raise ValueError('Data type was not defined')
1101
1106
1102 self.dtype = datatype_str
1107 self.dtype = datatype_str
1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1108 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1109 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1105 self.firstHeaderSize + self.basicHeaderSize * \
1110 self.firstHeaderSize + self.basicHeaderSize * \
1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1111 (self.processingHeaderObj.dataBlocksPerFile - 1)
1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1112 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1113 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1109 self.getBlockDimension()
1114 self.getBlockDimension()
1110
1115
1111 def __verifyFile(self, filename, msgFlag=True):
1116 def __verifyFile(self, filename, msgFlag=True):
1112
1117
1113 msg = None
1118 msg = None
1114
1119
1115 try:
1120 try:
1116 fp = open(filename, 'rb')
1121 fp = open(filename, 'rb')
1117 except IOError:
1122 except IOError:
1118
1123
1119 if msgFlag:
1124 if msgFlag:
1120 print("[Reading] File %s can't be opened" % (filename))
1125 print("[Reading] File %s can't be opened" % (filename))
1121
1126
1122 return False
1127 return False
1123
1128
1124 currentPosition = fp.tell()
1129 currentPosition = fp.tell()
1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1130 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1126
1131
1127 if neededSize == 0:
1132 if neededSize == 0:
1128 basicHeaderObj = BasicHeader(LOCALTIME)
1133 basicHeaderObj = BasicHeader(LOCALTIME)
1129 systemHeaderObj = SystemHeader()
1134 systemHeaderObj = SystemHeader()
1130 radarControllerHeaderObj = RadarControllerHeader()
1135 radarControllerHeaderObj = RadarControllerHeader()
1131 processingHeaderObj = ProcessingHeader()
1136 processingHeaderObj = ProcessingHeader()
1132
1137
1133 if not(basicHeaderObj.read(fp)):
1138 if not(basicHeaderObj.read(fp)):
1134 fp.close()
1139 fp.close()
1135 return False
1140 return False
1136
1141
1137 if not(systemHeaderObj.read(fp)):
1142 if not(systemHeaderObj.read(fp)):
1138 fp.close()
1143 fp.close()
1139 return False
1144 return False
1140
1145
1141 if not(radarControllerHeaderObj.read(fp)):
1146 if not(radarControllerHeaderObj.read(fp)):
1142 fp.close()
1147 fp.close()
1143 return False
1148 return False
1144
1149
1145 if not(processingHeaderObj.read(fp)):
1150 if not(processingHeaderObj.read(fp)):
1146 fp.close()
1151 fp.close()
1147 return False
1152 return False
1148
1153
1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1154 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1150 else:
1155 else:
1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1156 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1152
1157
1153 fp.close()
1158 fp.close()
1154
1159
1155 fileSize = os.path.getsize(filename)
1160 fileSize = os.path.getsize(filename)
1156 currentSize = fileSize - currentPosition
1161 currentSize = fileSize - currentPosition
1157
1162
1158 if currentSize < neededSize:
1163 if currentSize < neededSize:
1159 if msgFlag and (msg != None):
1164 if msgFlag and (msg != None):
1160 print(msg)
1165 print(msg)
1161 return False
1166 return False
1162
1167
1163 return True
1168 return True
1164
1169
1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1170 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1166
1171
1167 path_empty = True
1172 path_empty = True
1168
1173
1169 dateList = []
1174 dateList = []
1170 pathList = []
1175 pathList = []
1171
1176
1172 multi_path = path.split(',')
1177 multi_path = path.split(',')
1173
1178
1174 if not walk:
1179 if not walk:
1175
1180
1176 for single_path in multi_path:
1181 for single_path in multi_path:
1177
1182
1178 if not os.path.isdir(single_path):
1183 if not os.path.isdir(single_path):
1179 continue
1184 continue
1180
1185
1181 fileList = glob.glob1(single_path, "*" + ext)
1186 fileList = glob.glob1(single_path, "*" + ext)
1182
1187
1183 if not fileList:
1188 if not fileList:
1184 continue
1189 continue
1185
1190
1186 path_empty = False
1191 path_empty = False
1187
1192
1188 fileList.sort()
1193 fileList.sort()
1189
1194
1190 for thisFile in fileList:
1195 for thisFile in fileList:
1191
1196
1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1197 if not os.path.isfile(os.path.join(single_path, thisFile)):
1193 continue
1198 continue
1194
1199
1195 if not isRadarFile(thisFile):
1200 if not isRadarFile(thisFile):
1196 continue
1201 continue
1197
1202
1198 if not isFileInDateRange(thisFile, startDate, endDate):
1203 if not isFileInDateRange(thisFile, startDate, endDate):
1199 continue
1204 continue
1200
1205
1201 thisDate = getDateFromRadarFile(thisFile)
1206 thisDate = getDateFromRadarFile(thisFile)
1202
1207
1203 if thisDate in dateList:
1208 if thisDate in dateList:
1204 continue
1209 continue
1205
1210
1206 dateList.append(thisDate)
1211 dateList.append(thisDate)
1207 pathList.append(single_path)
1212 pathList.append(single_path)
1208
1213
1209 else:
1214 else:
1210 for single_path in multi_path:
1215 for single_path in multi_path:
1211
1216
1212 if not os.path.isdir(single_path):
1217 if not os.path.isdir(single_path):
1213 continue
1218 continue
1214
1219
1215 dirList = []
1220 dirList = []
1216
1221
1217 for thisPath in os.listdir(single_path):
1222 for thisPath in os.listdir(single_path):
1218
1223
1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1224 if not os.path.isdir(os.path.join(single_path, thisPath)):
1220 continue
1225 continue
1221
1226
1222 if not isRadarFolder(thisPath):
1227 if not isRadarFolder(thisPath):
1223 continue
1228 continue
1224
1229
1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1230 if not isFolderInDateRange(thisPath, startDate, endDate):
1226 continue
1231 continue
1227
1232
1228 dirList.append(thisPath)
1233 dirList.append(thisPath)
1229
1234
1230 if not dirList:
1235 if not dirList:
1231 continue
1236 continue
1232
1237
1233 dirList.sort()
1238 dirList.sort()
1234
1239
1235 for thisDir in dirList:
1240 for thisDir in dirList:
1236
1241
1237 datapath = os.path.join(single_path, thisDir, expLabel)
1242 datapath = os.path.join(single_path, thisDir, expLabel)
1238 fileList = glob.glob1(datapath, "*" + ext)
1243 fileList = glob.glob1(datapath, "*" + ext)
1239
1244
1240 if not fileList:
1245 if not fileList:
1241 continue
1246 continue
1242
1247
1243 path_empty = False
1248 path_empty = False
1244
1249
1245 thisDate = getDateFromRadarFolder(thisDir)
1250 thisDate = getDateFromRadarFolder(thisDir)
1246
1251
1247 pathList.append(datapath)
1252 pathList.append(datapath)
1248 dateList.append(thisDate)
1253 dateList.append(thisDate)
1249
1254
1250 dateList.sort()
1255 dateList.sort()
1251
1256
1252 if walk:
1257 if walk:
1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1258 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1254 else:
1259 else:
1255 pattern_path = multi_path[0]
1260 pattern_path = multi_path[0]
1256
1261
1257 if path_empty:
1262 if path_empty:
1258 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1263 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1259 else:
1264 else:
1260 if not dateList:
1265 if not dateList:
1261 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1266 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1262
1267
1263 if include_path:
1268 if include_path:
1264 return dateList, pathList
1269 return dateList, pathList
1265
1270
1266 return dateList
1271 return dateList
1267
1272
1268 def setup(self,
1273 def setup(self,
1269 path=None,
1274 path=None,
1270 startDate=None,
1275 startDate=None,
1271 endDate=None,
1276 endDate=None,
1272 startTime=datetime.time(0, 0, 0),
1277 startTime=datetime.time(0, 0, 0),
1273 endTime=datetime.time(23, 59, 59),
1278 endTime=datetime.time(23, 59, 59),
1274 set=None,
1279 set=None,
1275 expLabel="",
1280 expLabel="",
1276 ext=None,
1281 ext=None,
1277 online=False,
1282 online=False,
1278 delay=60,
1283 delay=60,
1279 walk=True,
1284 walk=True,
1280 getblock=False,
1285 getblock=False,
1281 nTxs=1,
1286 nTxs=1,
1282 realtime=False,
1287 realtime=False,
1283 blocksize=None,
1288 blocksize=None,
1284 blocktime=None,
1289 blocktime=None,
1285 skip=None,
1290 skip=None,
1286 cursor=None,
1291 cursor=None,
1287 warnings=True,
1292 warnings=True,
1288 verbose=True,
1293 verbose=True,
1289 server=None,
1294 server=None,
1290 format=None,
1295 format=None,
1291 oneDDict=None,
1296 oneDDict=None,
1292 twoDDict=None,
1297 twoDDict=None,
1293 independentParam=None):
1298 independentParam=None):
1294 if server is not None:
1299 if server is not None:
1295 if 'tcp://' in server:
1300 if 'tcp://' in server:
1296 address = server
1301 address = server
1297 else:
1302 else:
1298 address = 'ipc:///tmp/%s' % server
1303 address = 'ipc:///tmp/%s' % server
1299 self.server = address
1304 self.server = address
1300 self.context = zmq.Context()
1305 self.context = zmq.Context()
1301 self.receiver = self.context.socket(zmq.PULL)
1306 self.receiver = self.context.socket(zmq.PULL)
1302 self.receiver.connect(self.server)
1307 self.receiver.connect(self.server)
1303 time.sleep(0.5)
1308 time.sleep(0.5)
1304 print('[Starting] ReceiverData from {}'.format(self.server))
1309 print('[Starting] ReceiverData from {}'.format(self.server))
1305 else:
1310 else:
1306 self.server = None
1311 self.server = None
1307 if path == None:
1312 if path == None:
1308 raise ValueError("[Reading] The path is not valid")
1313 raise ValueError("[Reading] The path is not valid")
1309
1314
1310 if ext == None:
1315 if ext == None:
1311 ext = self.ext
1316 ext = self.ext
1312
1317
1313 if online:
1318 if online:
1314 print("[Reading] Searching files in online mode...")
1319 print("[Reading] Searching files in online mode...")
1315
1320
1316 for nTries in range(self.nTries):
1321 for nTries in range(self.nTries):
1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1322 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1323 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1319
1324
1320 if fullpath:
1325 if fullpath:
1321 break
1326 break
1322
1327
1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1328 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1324 sleep(self.delay)
1329 sleep(self.delay)
1325
1330
1326 if not(fullpath):
1331 if not(fullpath):
1327 self.dataOut.error = 'There isn\'t any valid file in {}'.format(path)
1332 self.dataOut.error = 'There isn\'t any valid file in {}'.format(path)
1328 return
1333 return
1329
1334
1330 self.year = year
1335 self.year = year
1331 self.doy = doy
1336 self.doy = doy
1332 self.set = set - 1
1337 self.set = set - 1
1333 self.path = path
1338 self.path = path
1334 self.foldercounter = foldercounter
1339 self.foldercounter = foldercounter
1335 last_set = None
1340 last_set = None
1336 else:
1341 else:
1337 print("[Reading] Searching files in offline mode ...")
1342 print("[Reading] Searching files in offline mode ...")
1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1343 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1339 startTime=startTime, endTime=endTime,
1344 startTime=startTime, endTime=endTime,
1340 set=set, expLabel=expLabel, ext=ext,
1345 set=set, expLabel=expLabel, ext=ext,
1341 walk=walk, cursor=cursor,
1346 walk=walk, cursor=cursor,
1342 skip=skip)
1347 skip=skip)
1343
1348
1344 if not(pathList):
1349 if not(pathList):
1345 self.fileIndex = -1
1350 self.fileIndex = -1
1346 self.pathList = []
1351 self.pathList = []
1347 self.filenameList = []
1352 self.filenameList = []
1348 return
1353 return
1349
1354
1350 self.fileIndex = -1
1355 self.fileIndex = -1
1351 self.pathList = pathList
1356 self.pathList = pathList
1352 self.filenameList = filenameList
1357 self.filenameList = filenameList
1353 file_name = os.path.basename(filenameList[-1])
1358 file_name = os.path.basename(filenameList[-1])
1354 basename, ext = os.path.splitext(file_name)
1359 basename, ext = os.path.splitext(file_name)
1355 last_set = int(basename[-3:])
1360 last_set = int(basename[-3:])
1356
1361
1357 self.online = online
1362 self.online = online
1358 self.realtime = realtime
1363 self.realtime = realtime
1359 self.delay = delay
1364 self.delay = delay
1360 ext = ext.lower()
1365 ext = ext.lower()
1361 self.ext = ext
1366 self.ext = ext
1362 self.getByBlock = getblock
1367 self.getByBlock = getblock
1363 self.nTxs = nTxs
1368 self.nTxs = nTxs
1364 self.startTime = startTime
1369 self.startTime = startTime
1365 self.endTime = endTime
1370 self.endTime = endTime
1366 self.endDate = endDate
1371 self.endDate = endDate
1367 self.startDate = startDate
1372 self.startDate = startDate
1368 # Added-----------------
1373 # Added-----------------
1369 self.selBlocksize = blocksize
1374 self.selBlocksize = blocksize
1370 self.selBlocktime = blocktime
1375 self.selBlocktime = blocktime
1371
1376
1372 # Verbose-----------
1377 # Verbose-----------
1373 self.verbose = verbose
1378 self.verbose = verbose
1374 self.warnings = warnings
1379 self.warnings = warnings
1375
1380
1376 if not(self.setNextFile()):
1381 if not(self.setNextFile()):
1377 if (startDate != None) and (endDate != None):
1382 if (startDate != None) and (endDate != None):
1378 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1383 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1379 elif startDate != None:
1384 elif startDate != None:
1380 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1385 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1381 else:
1386 else:
1382 print("[Reading] No files")
1387 print("[Reading] No files")
1383
1388
1384 self.fileIndex = -1
1389 self.fileIndex = -1
1385 self.pathList = []
1390 self.pathList = []
1386 self.filenameList = []
1391 self.filenameList = []
1387 return
1392 return
1388
1393
1389 # self.getBasicHeader()
1394 # self.getBasicHeader()
1390
1395
1391 if last_set != None:
1396 if last_set != None:
1392 self.dataOut.last_block = last_set * \
1397 self.dataOut.last_block = last_set * \
1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1398 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1394 return
1399 return
1395
1400
1396 def getBasicHeader(self):
1401 def getBasicHeader(self):
1397
1402
1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1403 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1404 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1400
1405
1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1406 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1402
1407
1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1408 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1404
1409
1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1410 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1406
1411
1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1412 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1408
1413
1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1414 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1410
1415
1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1416 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1412
1417
1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1418 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1414
1419
1415 def getFirstHeader(self):
1420 def getFirstHeader(self):
1416
1421
1417 raise NotImplementedError
1422 raise NotImplementedError
1418
1423
1419 def getData(self):
1424 def getData(self):
1420
1425
1421 raise NotImplementedError
1426 raise NotImplementedError
1422
1427
1423 def hasNotDataInBuffer(self):
1428 def hasNotDataInBuffer(self):
1424
1429
1425 raise NotImplementedError
1430 raise NotImplementedError
1426
1431
1427 def readBlock(self):
1432 def readBlock(self):
1428
1433
1429 raise NotImplementedError
1434 raise NotImplementedError
1430
1435
1431 def isEndProcess(self):
1436 def isEndProcess(self):
1432
1437
1433 return self.flagNoMoreFiles
1438 return self.flagNoMoreFiles
1434
1439
1435 def printReadBlocks(self):
1440 def printReadBlocks(self):
1436
1441
1437 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1442 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1438
1443
1439 def printTotalBlocks(self):
1444 def printTotalBlocks(self):
1440
1445
1441 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1446 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1442
1447
1443 def printNumberOfBlock(self):
1448 def printNumberOfBlock(self):
1444 'SPAM!'
1449 'SPAM!'
1445
1450
1446 # if self.flagIsNewBlock:
1451 # if self.flagIsNewBlock:
1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1452 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1448 # self.processingHeaderObj.dataBlocksPerFile,
1453 # self.processingHeaderObj.dataBlocksPerFile,
1449 # self.dataOut.datatime.ctime())
1454 # self.dataOut.datatime.ctime())
1450
1455
1451 def printInfo(self):
1456 def printInfo(self):
1452
1457
1453 if self.__printInfo == False:
1458 if self.__printInfo == False:
1454 return
1459 return
1455
1460
1456 self.basicHeaderObj.printInfo()
1461 self.basicHeaderObj.printInfo()
1457 self.systemHeaderObj.printInfo()
1462 self.systemHeaderObj.printInfo()
1458 self.radarControllerHeaderObj.printInfo()
1463 self.radarControllerHeaderObj.printInfo()
1459 self.processingHeaderObj.printInfo()
1464 self.processingHeaderObj.printInfo()
1460
1465
1461 self.__printInfo = False
1466 self.__printInfo = False
1462
1467
1463 def run(self,
1468 def run(self,
1464 path=None,
1469 path=None,
1465 startDate=None,
1470 startDate=None,
1466 endDate=None,
1471 endDate=None,
1467 startTime=datetime.time(0, 0, 0),
1472 startTime=datetime.time(0, 0, 0),
1468 endTime=datetime.time(23, 59, 59),
1473 endTime=datetime.time(23, 59, 59),
1469 set=None,
1474 set=None,
1470 expLabel="",
1475 expLabel="",
1471 ext=None,
1476 ext=None,
1472 online=False,
1477 online=False,
1473 delay=60,
1478 delay=60,
1474 walk=True,
1479 walk=True,
1475 getblock=False,
1480 getblock=False,
1476 nTxs=1,
1481 nTxs=1,
1477 realtime=False,
1482 realtime=False,
1478 blocksize=None,
1483 blocksize=None,
1479 blocktime=None,
1484 blocktime=None,
1480 skip=None,
1485 skip=None,
1481 cursor=None,
1486 cursor=None,
1482 warnings=True,
1487 warnings=True,
1483 server=None,
1488 server=None,
1484 verbose=True,
1489 verbose=True,
1485 format=None,
1490 format=None,
1486 oneDDict=None,
1491 oneDDict=None,
1487 twoDDict=None,
1492 twoDDict=None,
1488 independentParam=None, **kwargs):
1493 independentParam=None, **kwargs):
1489
1494
1490 if not(self.isConfig):
1495 if not(self.isConfig):
1491 self.setup(path=path,
1496 self.setup(path=path,
1492 startDate=startDate,
1497 startDate=startDate,
1493 endDate=endDate,
1498 endDate=endDate,
1494 startTime=startTime,
1499 startTime=startTime,
1495 endTime=endTime,
1500 endTime=endTime,
1496 set=set,
1501 set=set,
1497 expLabel=expLabel,
1502 expLabel=expLabel,
1498 ext=ext,
1503 ext=ext,
1499 online=online,
1504 online=online,
1500 delay=delay,
1505 delay=delay,
1501 walk=walk,
1506 walk=walk,
1502 getblock=getblock,
1507 getblock=getblock,
1503 nTxs=nTxs,
1508 nTxs=nTxs,
1504 realtime=realtime,
1509 realtime=realtime,
1505 blocksize=blocksize,
1510 blocksize=blocksize,
1506 blocktime=blocktime,
1511 blocktime=blocktime,
1507 skip=skip,
1512 skip=skip,
1508 cursor=cursor,
1513 cursor=cursor,
1509 warnings=warnings,
1514 warnings=warnings,
1510 server=server,
1515 server=server,
1511 verbose=verbose,
1516 verbose=verbose,
1512 format=format,
1517 format=format,
1513 oneDDict=oneDDict,
1518 oneDDict=oneDDict,
1514 twoDDict=twoDDict,
1519 twoDDict=twoDDict,
1515 independentParam=independentParam)
1520 independentParam=independentParam)
1516 self.isConfig = True
1521 self.isConfig = True
1517 if server is None:
1522 if server is None:
1518 self.getData()
1523 self.getData()
1519 else:
1524 else:
1520 self.getFromServer()
1525 self.getFromServer()
1521
1526
1522
1527
1523 class JRODataWriter(JRODataIO):
1528 class JRODataWriter(JRODataIO):
1524
1529
1525 """
1530 """
1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1531 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1527 de los datos siempre se realiza por bloques.
1532 de los datos siempre se realiza por bloques.
1528 """
1533 """
1529
1534
1530 blockIndex = 0
1535 blockIndex = 0
1531
1536
1532 path = None
1537 path = None
1533
1538
1534 setFile = None
1539 setFile = None
1535
1540
1536 profilesPerBlock = None
1541 profilesPerBlock = None
1537
1542
1538 blocksPerFile = None
1543 blocksPerFile = None
1539
1544
1540 nWriteBlocks = 0
1545 nWriteBlocks = 0
1541
1546
1542 fileDate = None
1547 fileDate = None
1543
1548
1544 def __init__(self, dataOut=None):
1549 def __init__(self, dataOut=None):
1545 raise NotImplementedError
1550 raise NotImplementedError
1546
1551
1547 def hasAllDataInBuffer(self):
1552 def hasAllDataInBuffer(self):
1548 raise NotImplementedError
1553 raise NotImplementedError
1549
1554
1550 def setBlockDimension(self):
1555 def setBlockDimension(self):
1551 raise NotImplementedError
1556 raise NotImplementedError
1552
1557
1553 def writeBlock(self):
1558 def writeBlock(self):
1554 raise NotImplementedError
1559 raise NotImplementedError
1555
1560
1556 def putData(self):
1561 def putData(self):
1557 raise NotImplementedError
1562 raise NotImplementedError
1558
1563
1559 def getProcessFlags(self):
1564 def getProcessFlags(self):
1560
1565
1561 processFlags = 0
1566 processFlags = 0
1562
1567
1563 dtype_index = get_dtype_index(self.dtype)
1568 dtype_index = get_dtype_index(self.dtype)
1564 procflag_dtype = get_procflag_dtype(dtype_index)
1569 procflag_dtype = get_procflag_dtype(dtype_index)
1565
1570
1566 processFlags += procflag_dtype
1571 processFlags += procflag_dtype
1567
1572
1568 if self.dataOut.flagDecodeData:
1573 if self.dataOut.flagDecodeData:
1569 processFlags += PROCFLAG.DECODE_DATA
1574 processFlags += PROCFLAG.DECODE_DATA
1570
1575
1571 if self.dataOut.flagDeflipData:
1576 if self.dataOut.flagDeflipData:
1572 processFlags += PROCFLAG.DEFLIP_DATA
1577 processFlags += PROCFLAG.DEFLIP_DATA
1573
1578
1574 if self.dataOut.code is not None:
1579 if self.dataOut.code is not None:
1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1580 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1576
1581
1577 if self.dataOut.nCohInt > 1:
1582 if self.dataOut.nCohInt > 1:
1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1583 processFlags += PROCFLAG.COHERENT_INTEGRATION
1579
1584
1580 if self.dataOut.type == "Spectra":
1585 if self.dataOut.type == "Spectra":
1581 if self.dataOut.nIncohInt > 1:
1586 if self.dataOut.nIncohInt > 1:
1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1587 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1583
1588
1584 if self.dataOut.data_dc is not None:
1589 if self.dataOut.data_dc is not None:
1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1590 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1586
1591
1587 if self.dataOut.flagShiftFFT:
1592 if self.dataOut.flagShiftFFT:
1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1593 processFlags += PROCFLAG.SHIFT_FFT_DATA
1589
1594
1590 return processFlags
1595 return processFlags
1591
1596
1592 def setBasicHeader(self):
1597 def setBasicHeader(self):
1593
1598
1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1599 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1595 self.basicHeaderObj.version = self.versionFile
1600 self.basicHeaderObj.version = self.versionFile
1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1601 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1597
1602
1598 utc = numpy.floor(self.dataOut.utctime)
1603 utc = numpy.floor(self.dataOut.utctime)
1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1604 milisecond = (self.dataOut.utctime - utc) * 1000.0
1600
1605
1601 self.basicHeaderObj.utc = utc
1606 self.basicHeaderObj.utc = utc
1602 self.basicHeaderObj.miliSecond = milisecond
1607 self.basicHeaderObj.miliSecond = milisecond
1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1608 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1609 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1610 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1606
1611
1607 def setFirstHeader(self):
1612 def setFirstHeader(self):
1608 """
1613 """
1609 Obtiene una copia del First Header
1614 Obtiene una copia del First Header
1610
1615
1611 Affected:
1616 Affected:
1612
1617
1613 self.basicHeaderObj
1618 self.basicHeaderObj
1614 self.systemHeaderObj
1619 self.systemHeaderObj
1615 self.radarControllerHeaderObj
1620 self.radarControllerHeaderObj
1616 self.processingHeaderObj self.
1621 self.processingHeaderObj self.
1617
1622
1618 Return:
1623 Return:
1619 None
1624 None
1620 """
1625 """
1621
1626
1622 raise NotImplementedError
1627 raise NotImplementedError
1623
1628
1624 def __writeFirstHeader(self):
1629 def __writeFirstHeader(self):
1625 """
1630 """
1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1631 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1627
1632
1628 Affected:
1633 Affected:
1629 __dataType
1634 __dataType
1630
1635
1631 Return:
1636 Return:
1632 None
1637 None
1633 """
1638 """
1634
1639
1635 # CALCULAR PARAMETROS
1640 # CALCULAR PARAMETROS
1636
1641
1637 sizeLongHeader = self.systemHeaderObj.size + \
1642 sizeLongHeader = self.systemHeaderObj.size + \
1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1643 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1644 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1640
1645
1641 self.basicHeaderObj.write(self.fp)
1646 self.basicHeaderObj.write(self.fp)
1642 self.systemHeaderObj.write(self.fp)
1647 self.systemHeaderObj.write(self.fp)
1643 self.radarControllerHeaderObj.write(self.fp)
1648 self.radarControllerHeaderObj.write(self.fp)
1644 self.processingHeaderObj.write(self.fp)
1649 self.processingHeaderObj.write(self.fp)
1645
1650
1646 def __setNewBlock(self):
1651 def __setNewBlock(self):
1647 """
1652 """
1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1653 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1649
1654
1650 Return:
1655 Return:
1651 0 : si no pudo escribir nada
1656 0 : si no pudo escribir nada
1652 1 : Si escribio el Basic el First Header
1657 1 : Si escribio el Basic el First Header
1653 """
1658 """
1654 if self.fp == None:
1659 if self.fp == None:
1655 self.setNextFile()
1660 self.setNextFile()
1656
1661
1657 if self.flagIsNewFile:
1662 if self.flagIsNewFile:
1658 return 1
1663 return 1
1659
1664
1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1665 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1661 self.basicHeaderObj.write(self.fp)
1666 self.basicHeaderObj.write(self.fp)
1662 return 1
1667 return 1
1663
1668
1664 if not(self.setNextFile()):
1669 if not(self.setNextFile()):
1665 return 0
1670 return 0
1666
1671
1667 return 1
1672 return 1
1668
1673
1669 def writeNextBlock(self):
1674 def writeNextBlock(self):
1670 """
1675 """
1671 Selecciona el bloque siguiente de datos y los escribe en un file
1676 Selecciona el bloque siguiente de datos y los escribe en un file
1672
1677
1673 Return:
1678 Return:
1674 0 : Si no hizo pudo escribir el bloque de datos
1679 0 : Si no hizo pudo escribir el bloque de datos
1675 1 : Si no pudo escribir el bloque de datos
1680 1 : Si no pudo escribir el bloque de datos
1676 """
1681 """
1677 if not(self.__setNewBlock()):
1682 if not(self.__setNewBlock()):
1678 return 0
1683 return 0
1679
1684
1680 self.writeBlock()
1685 self.writeBlock()
1681
1686
1682 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1687 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1683 self.processingHeaderObj.dataBlocksPerFile))
1688 self.processingHeaderObj.dataBlocksPerFile))
1684
1689
1685 return 1
1690 return 1
1686
1691
1687 def setNextFile(self):
1692 def setNextFile(self):
1688 """
1693 """
1689 Determina el siguiente file que sera escrito
1694 Determina el siguiente file que sera escrito
1690
1695
1691 Affected:
1696 Affected:
1692 self.filename
1697 self.filename
1693 self.subfolder
1698 self.subfolder
1694 self.fp
1699 self.fp
1695 self.setFile
1700 self.setFile
1696 self.flagIsNewFile
1701 self.flagIsNewFile
1697
1702
1698 Return:
1703 Return:
1699 0 : Si el archivo no puede ser escrito
1704 0 : Si el archivo no puede ser escrito
1700 1 : Si el archivo esta listo para ser escrito
1705 1 : Si el archivo esta listo para ser escrito
1701 """
1706 """
1702 ext = self.ext
1707 ext = self.ext
1703 path = self.path
1708 path = self.path
1704
1709
1705 if self.fp != None:
1710 if self.fp != None:
1706 self.fp.close()
1711 self.fp.close()
1707
1712
1708 timeTuple = time.localtime(self.dataOut.utctime)
1713 timeTuple = time.localtime(self.dataOut.utctime)
1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1714 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1710
1715
1711 fullpath = os.path.join(path, subfolder)
1716 fullpath = os.path.join(path, subfolder)
1712 setFile = self.setFile
1717 setFile = self.setFile
1713
1718
1714 if not(os.path.exists(fullpath)):
1719 if not(os.path.exists(fullpath)):
1715 os.mkdir(fullpath)
1720 os.mkdir(fullpath)
1716 setFile = -1 # inicializo mi contador de seteo
1721 setFile = -1 # inicializo mi contador de seteo
1717 else:
1722 else:
1718 filesList = os.listdir(fullpath)
1723 filesList = os.listdir(fullpath)
1719 if len(filesList) > 0:
1724 if len(filesList) > 0:
1720 filesList = sorted(filesList, key=str.lower)
1725 filesList = sorted(filesList, key=str.lower)
1721 filen = filesList[-1]
1726 filen = filesList[-1]
1722 # el filename debera tener el siguiente formato
1727 # el filename debera tener el siguiente formato
1723 # 0 1234 567 89A BCDE (hex)
1728 # 0 1234 567 89A BCDE (hex)
1724 # x YYYY DDD SSS .ext
1729 # x YYYY DDD SSS .ext
1725 if isNumber(filen[8:11]):
1730 if isNumber(filen[8:11]):
1726 # inicializo mi contador de seteo al seteo del ultimo file
1731 # inicializo mi contador de seteo al seteo del ultimo file
1727 setFile = int(filen[8:11])
1732 setFile = int(filen[8:11])
1728 else:
1733 else:
1729 setFile = -1
1734 setFile = -1
1730 else:
1735 else:
1731 setFile = -1 # inicializo mi contador de seteo
1736 setFile = -1 # inicializo mi contador de seteo
1732
1737
1733 setFile += 1
1738 setFile += 1
1734
1739
1735 # If this is a new day it resets some values
1740 # If this is a new day it resets some values
1736 if self.dataOut.datatime.date() > self.fileDate:
1741 if self.dataOut.datatime.date() > self.fileDate:
1737 setFile = 0
1742 setFile = 0
1738 self.nTotalBlocks = 0
1743 self.nTotalBlocks = 0
1739
1744
1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1745 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1746 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1742
1747
1743 filename = os.path.join(path, subfolder, filen)
1748 filename = os.path.join(path, subfolder, filen)
1744
1749
1745 fp = open(filename, 'wb')
1750 fp = open(filename, 'wb')
1746
1751
1747 self.blockIndex = 0
1752 self.blockIndex = 0
1748
1753
1749 # guardando atributos
1754 # guardando atributos
1750 self.filename = filename
1755 self.filename = filename
1751 self.subfolder = subfolder
1756 self.subfolder = subfolder
1752 self.fp = fp
1757 self.fp = fp
1753 self.setFile = setFile
1758 self.setFile = setFile
1754 self.flagIsNewFile = 1
1759 self.flagIsNewFile = 1
1755 self.fileDate = self.dataOut.datatime.date()
1760 self.fileDate = self.dataOut.datatime.date()
1756
1761
1757 self.setFirstHeader()
1762 self.setFirstHeader()
1758
1763
1759 print('[Writing] Opening file: %s' % self.filename)
1764 print('[Writing] Opening file: %s' % self.filename)
1760
1765
1761 self.__writeFirstHeader()
1766 self.__writeFirstHeader()
1762
1767
1763 return 1
1768 return 1
1764
1769
1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1770 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1766 """
1771 """
1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1772 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1768
1773
1769 Inputs:
1774 Inputs:
1770 path : directory where data will be saved
1775 path : directory where data will be saved
1771 profilesPerBlock : number of profiles per block
1776 profilesPerBlock : number of profiles per block
1772 set : initial file set
1777 set : initial file set
1773 datatype : An integer number that defines data type:
1778 datatype : An integer number that defines data type:
1774 0 : int8 (1 byte)
1779 0 : int8 (1 byte)
1775 1 : int16 (2 bytes)
1780 1 : int16 (2 bytes)
1776 2 : int32 (4 bytes)
1781 2 : int32 (4 bytes)
1777 3 : int64 (8 bytes)
1782 3 : int64 (8 bytes)
1778 4 : float32 (4 bytes)
1783 4 : float32 (4 bytes)
1779 5 : double64 (8 bytes)
1784 5 : double64 (8 bytes)
1780
1785
1781 Return:
1786 Return:
1782 0 : Si no realizo un buen seteo
1787 0 : Si no realizo un buen seteo
1783 1 : Si realizo un buen seteo
1788 1 : Si realizo un buen seteo
1784 """
1789 """
1785
1790
1786 if ext == None:
1791 if ext == None:
1787 ext = self.ext
1792 ext = self.ext
1788
1793
1789 self.ext = ext.lower()
1794 self.ext = ext.lower()
1790
1795
1791 self.path = path
1796 self.path = path
1792
1797
1793 if set is None:
1798 if set is None:
1794 self.setFile = -1
1799 self.setFile = -1
1795 else:
1800 else:
1796 self.setFile = set - 1
1801 self.setFile = set - 1
1797
1802
1798 self.blocksPerFile = blocksPerFile
1803 self.blocksPerFile = blocksPerFile
1799
1804
1800 self.profilesPerBlock = profilesPerBlock
1805 self.profilesPerBlock = profilesPerBlock
1801
1806
1802 self.dataOut = dataOut
1807 self.dataOut = dataOut
1803 self.fileDate = self.dataOut.datatime.date()
1808 self.fileDate = self.dataOut.datatime.date()
1804 # By default
1809 # By default
1805 self.dtype = self.dataOut.dtype
1810 self.dtype = self.dataOut.dtype
1806
1811
1807 if datatype is not None:
1812 if datatype is not None:
1808 self.dtype = get_numpy_dtype(datatype)
1813 self.dtype = get_numpy_dtype(datatype)
1809
1814
1810 if not(self.setNextFile()):
1815 if not(self.setNextFile()):
1811 print("[Writing] There isn't a next file")
1816 print("[Writing] There isn't a next file")
1812 return 0
1817 return 0
1813
1818
1814 self.setBlockDimension()
1819 self.setBlockDimension()
1815
1820
1816 return 1
1821 return 1
1817
1822
1818 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1823 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1819
1824
1820 if not(self.isConfig):
1825 if not(self.isConfig):
1821
1826
1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1827 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1823 set=set, ext=ext, datatype=datatype, **kwargs)
1828 set=set, ext=ext, datatype=datatype, **kwargs)
1824 self.isConfig = True
1829 self.isConfig = True
1825
1830
1826 self.dataOut = dataOut
1831 self.dataOut = dataOut
1827 self.putData()
1832 self.putData()
1828 return self.dataOut No newline at end of file
1833 return self.dataOut
General Comments 0
You need to be logged in to leave comments. Login now