##// END OF EJS Templates
schain-jc
RSS
Add reading online mode for BLTR parameters
Juan C. Espinoza -
r1215:785b27922a3d
parent child
Show More
Side by Side Unified
Context file:
r1215:785b27922a3d Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,368 +1,409
1 1 '''
2 2 Created on Nov 9, 2016
3 3
4 4 @author: roj- LouVD
5 5 '''
6 6
7 7
8 8 import os
9 9 import sys
10 10 import time
11 11 import glob
12 12 import datetime
13 13
14 14 import numpy
15 15
16 16 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator
17 17 from schainpy.model.data.jrodata import Parameters
18 18 from schainpy.model.io.jroIO_base import JRODataReader, isNumber
19 19 from schainpy.utils import log
20 20
21 21 FILE_HEADER_STRUCTURE = numpy.dtype([
22 22 ('FMN', '<u4'),
23 23 ('nrec', '<u4'),
24 24 ('fr_offset', '<u4'),
25 25 ('id', '<u4'),
26 26 ('site', 'u1', (32,))
27 27 ])
28 28
29 29 REC_HEADER_STRUCTURE = numpy.dtype([
30 30 ('rmn', '<u4'),
31 31 ('rcounter', '<u4'),
32 32 ('nr_offset', '<u4'),
33 33 ('tr_offset', '<u4'),
34 34 ('time', '<u4'),
35 35 ('time_msec', '<u4'),
36 36 ('tag', 'u1', (32,)),
37 37 ('comments', 'u1', (32,)),
38 38 ('lat', '<f4'),
39 39 ('lon', '<f4'),
40 40 ('gps_status', '<u4'),
41 41 ('freq', '<u4'),
42 42 ('freq0', '<u4'),
43 43 ('nchan', '<u4'),
44 44 ('delta_r', '<u4'),
45 45 ('nranges', '<u4'),
46 46 ('r0', '<u4'),
47 47 ('prf', '<u4'),
48 48 ('ncoh', '<u4'),
49 49 ('npoints', '<u4'),
50 50 ('polarization', '<i4'),
51 51 ('rx_filter', '<u4'),
52 52 ('nmodes', '<u4'),
53 53 ('dmode_index', '<u4'),
54 54 ('dmode_rngcorr', '<u4'),
55 55 ('nrxs', '<u4'),
56 56 ('acf_length', '<u4'),
57 57 ('acf_lags', '<u4'),
58 58 ('sea_to_atmos', '<f4'),
59 59 ('sea_notch', '<u4'),
60 60 ('lh_sea', '<u4'),
61 61 ('hh_sea', '<u4'),
62 62 ('nbins_sea', '<u4'),
63 63 ('min_snr', '<f4'),
64 64 ('min_cc', '<f4'),
65 65 ('max_time_diff', '<f4')
66 66 ])
67 67
68 68 DATA_STRUCTURE = numpy.dtype([
69 69 ('range', '<u4'),
70 70 ('status', '<u4'),
71 71 ('zonal', '<f4'),
72 72 ('meridional', '<f4'),
73 73 ('vertical', '<f4'),
74 74 ('zonal_a', '<f4'),
75 75 ('meridional_a', '<f4'),
76 76 ('corrected_fading', '<f4'), # seconds
77 77 ('uncorrected_fading', '<f4'), # seconds
78 78 ('time_diff', '<f4'),
79 79 ('major_axis', '<f4'),
80 80 ('axial_ratio', '<f4'),
81 81 ('orientation', '<f4'),
82 82 ('sea_power', '<u4'),
83 83 ('sea_algorithm', '<u4')
84 84 ])
85 85
86 86 @MPDecorator
87 87 class BLTRParamReader(JRODataReader, ProcessingUnit):
88 88 '''
89 89 Boundary Layer and Tropospheric Radar (BLTR) reader, Wind velocities and SNR from *.sswma files
90 90 '''
91 91
92 92 ext = '.sswma'
93 93
94 94 def __init__(self):
95 95
96 96 ProcessingUnit.__init__(self)
97 97
98 98 self.dataOut = Parameters()
99 99 self.counter_records = 0
100 100 self.flagNoMoreFiles = 0
101 101 self.isConfig = False
102 102 self.filename = None
103 103
104 104 def setup(self,
105 105 path=None,
106 106 startDate=None,
107 107 endDate=None,
108 108 ext=None,
109 109 startTime=datetime.time(0, 0, 0),
110 110 endTime=datetime.time(23, 59, 59),
111 111 timezone=0,
112 112 status_value=0,
113 113 **kwargs):
114 114 self.path = path
115 115 self.startDate = startDate
116 116 self.endDate = endDate
117 117 self.startTime = startTime
118 118 self.endTime = endTime
119 119 self.status_value = status_value
120 120 self.datatime = datetime.datetime(1900,1,1)
121 self.delay = kwargs.get('delay', 10)
122 self.online = kwargs.get('online', False)
123 self.nTries = kwargs.get('nTries', 3)
121 124
122 125 if self.path is None:
123 126 raise ValueError("The path is not valid")
124 127
125 128 if ext is None:
126 129 ext = self.ext
127 130
128 self.search_files(self.path, startDate, endDate, ext)
131 self.fileList = self.search_files(self.path, startDate, endDate, ext)
129 132 self.timezone = timezone
130 133 self.fileIndex = 0
131 134
132 135 if not self.fileList:
133 136 raise Warning("There is no files matching these date in the folder: %s. \n Check 'startDate' and 'endDate' " % (
134 137 path))
135 138
136 139 self.setNextFile()
137 140
141 def search_last_file(self):
142 '''
143 Get last file and add it to the list
144 '''
145
146 for n in range(self.nTries):
147 if n > 0:
148 log.warning(
149 "Waiting %0.2f sec for the next file, try %03d ..." % (self.delay, self.nTries + 1),
150 self.name
151 )
152 time.sleep(self.delay)
153 file_list = os.listdir(self.path)
154 file_list.sort()
155 if file_list:
156 if self.filename:
157 if file_list[-1] not in self.filename:
158 return file_list[-1]
159 else:
160 continue
161 return file_list[-1]
162 return 0
163
138 164 def search_files(self, path, startDate, endDate, ext):
139 165 '''
140 166 Searching for BLTR rawdata file in path
141 167 Creating a list of file to proces included in [startDate,endDate]
142 168
143 169 Input:
144 170 path - Path to find BLTR rawdata files
145 171 startDate - Select file from this date
146 172 enDate - Select file until this date
147 173 ext - Extension of the file to read
148 174 '''
149 175
150 176 log.success('Searching files in {} '.format(path), 'BLTRParamReader')
151 177 foldercounter = 0
152 178 fileList0 = glob.glob1(path, "*%s" % ext)
153 179 fileList0.sort()
154 180
155 self.fileList = []
156 self.dateFileList = []
181 #self.fileList = []
182 #self.dateFileList = []
157 183
158 184 for thisFile in fileList0:
159 185 year = thisFile[-14:-10]
160 186 if not isNumber(year):
161 187 continue
162 188
163 189 month = thisFile[-10:-8]
164 190 if not isNumber(month):
165 191 continue
166 192
167 193 day = thisFile[-8:-6]
168 194 if not isNumber(day):
169 195 continue
170 196
171 197 year, month, day = int(year), int(month), int(day)
172 198 dateFile = datetime.date(year, month, day)
173 199
174 200 if (startDate > dateFile) or (endDate < dateFile):
175 201 continue
176 202
177 self.fileList.append(thisFile)
178 self.dateFileList.append(dateFile)
203 yield thisFile
204 # self.dateFileList.append(dateFile)
179 205
180 206 return
181 207
182 208 def setNextFile(self):
183 209
184 file_id = self.fileIndex
185
186 if file_id == len(self.fileList):
187 self.flagNoMoreFiles = 1
188 return 0
189
190 log.success('Opening {}'.format(self.fileList[file_id]), 'BLTRParamReader')
191 filename = os.path.join(self.path, self.fileList[file_id])
210 if self.online:
211 filename = self.search_last_file()
212 if not filename:
213 self.flagNoMoreFiles = 1
214 return 0
215 else:
216 try:
217 filename = next(self.fileList)
218 except StopIteration:
219 print('Noooo files')
220 self.flagNoMoreFiles = 1
221 return 0
222
223 log.success('Opening {}'.format(filename), 'BLTRParamReader')
192 224
193 225 dirname, name = os.path.split(filename)
194 226 # 'peru2' ---> Piura - 'peru1' ---> Huancayo or Porcuya
195 self.siteFile = name.split('.')[0]
227 self.siteFile = filename.split('.')[0]
196 228 if self.filename is not None:
197 229 self.fp.close()
198 self.filename = filename
230 self.filename = os.path.join(self.path, filename)
199 231 self.fp = open(self.filename, 'rb')
200 232 self.header_file = numpy.fromfile(self.fp, FILE_HEADER_STRUCTURE, 1)
201 233 self.nrecords = self.header_file['nrec'][0]
202 234 self.sizeOfFile = os.path.getsize(self.filename)
203 235 self.counter_records = 0
204 236 self.flagIsNewFile = 0
205 237 self.fileIndex += 1
206 238
207 239 return 1
208 240
209 241 def readNextBlock(self):
210 242
211 243 while True:
212 244 if self.counter_records == self.nrecords:
213 245 self.flagIsNewFile = 1
214 246 if not self.setNextFile():
215 247 return 0
216 248
217 self.readBlock()
249 try:
250 pointer = self.fp.tell()
251 self.readBlock()
252 except:
253 if self.waitDataBlock(pointer, 38512) == 1:
254 continue
255 else:
256 if not self.setNextFile():
257 return 0
218 258
219 259 if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
220 260 (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
221 261 log.warning(
222 262 'Reading Record No. {}/{} -> {} [Skipping]'.format(
223 263 self.counter_records,
224 264 self.nrecords,
225 265 self.datatime.ctime()),
226 266 'BLTRParamReader')
227 267 continue
228 268 break
229 269
230 270 log.log('Reading Record No. {}/{} -> {}'.format(
231 271 self.counter_records,
232 272 self.nrecords,
233 273 self.datatime.ctime()), 'BLTRParamReader')
234 274
235 275 return 1
236 276
237 277 def readBlock(self):
238 278
239 279 pointer = self.fp.tell()
240 280 header_rec = numpy.fromfile(self.fp, REC_HEADER_STRUCTURE, 1)
241 281 self.nchannels = int(header_rec['nchan'][0] / 2)
242 282 self.kchan = header_rec['nrxs'][0]
243 283 self.nmodes = header_rec['nmodes'][0]
244 284 self.nranges = header_rec['nranges'][0]
245 285 self.fp.seek(pointer)
246 286 self.height = numpy.empty((self.nmodes, self.nranges))
247 287 self.snr = numpy.empty((self.nmodes, int(self.nchannels), self.nranges))
248 288 self.buffer = numpy.empty((self.nmodes, 3, self.nranges))
249 289 self.flagDiscontinuousBlock = 0
250 290
251 291 for mode in range(self.nmodes):
252 292 self.readHeader()
253 293 data = self.readData()
254 294 self.height[mode] = (data[0] - self.correction) / 1000.
255 295 self.buffer[mode] = data[1]
256 296 self.snr[mode] = data[2]
257 297
258 298 self.counter_records = self.counter_records + self.nmodes
259 299
260 300 return
261 301
262 302 def readHeader(self):
263 303 '''
264 304 RecordHeader of BLTR rawdata file
265 305 '''
266 306
267 307 header_structure = numpy.dtype(
268 308 REC_HEADER_STRUCTURE.descr + [
269 309 ('antenna_coord', 'f4', (2, int(self.nchannels))),
270 310 ('rx_gains', 'u4', (int(self.nchannels),)),
271 311 ('rx_analysis', 'u4', (int(self.nchannels),))
272 312 ]
273 313 )
274 314
275 315 self.header_rec = numpy.fromfile(self.fp, header_structure, 1)
276 316 self.lat = self.header_rec['lat'][0]
277 317 self.lon = self.header_rec['lon'][0]
278 318 self.delta = self.header_rec['delta_r'][0]
279 319 self.correction = self.header_rec['dmode_rngcorr'][0]
280 320 self.imode = self.header_rec['dmode_index'][0]
281 321 self.antenna = self.header_rec['antenna_coord']
282 322 self.rx_gains = self.header_rec['rx_gains']
283 323 self.time = self.header_rec['time'][0]
284 324 dt = datetime.datetime.utcfromtimestamp(self.time)
285 325 if dt.date()>self.datatime.date():
286 326 self.flagDiscontinuousBlock = 1
287 327 self.datatime = dt
288 328
289 329 def readData(self):
290 330 '''
291 331 Reading and filtering data block record of BLTR rawdata file, filtering is according to status_value.
292 332
293 333 Input:
294 334 status_value - Array data is set to NAN for values that are not equal to status_value
295 335
296 336 '''
297 337 self.nchannels = int(self.nchannels)
298 338
299 339 data_structure = numpy.dtype(
300 340 DATA_STRUCTURE.descr + [
301 341 ('rx_saturation', 'u4', (self.nchannels,)),
302 342 ('chan_offset', 'u4', (2 * self.nchannels,)),
303 343 ('rx_amp', 'u4', (self.nchannels,)),
304 344 ('rx_snr', 'f4', (self.nchannels,)),
305 345 ('cross_snr', 'f4', (self.kchan,)),
306 346 ('sea_power_relative', 'f4', (self.kchan,))]
307 347 )
308 348
309 349 data = numpy.fromfile(self.fp, data_structure, self.nranges)
310 350
311 351 height = data['range']
312 352 winds = numpy.array(
313 353 (data['zonal'], data['meridional'], data['vertical']))
314 354 snr = data['rx_snr'].T
315 355
316 356 winds[numpy.where(winds == -9999.)] = numpy.nan
317 357 winds[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
318 358 snr[numpy.where(snr == -9999.)] = numpy.nan
319 359 snr[:, numpy.where(data['status'] != self.status_value)] = numpy.nan
320 360 snr = numpy.power(10, snr / 10)
321 361
322 362 return height, winds, snr
323 363
324 364 def set_output(self):
325 365 '''
326 366 Storing data from databuffer to dataOut object
327 367 '''
328 368
329 369 self.dataOut.data_SNR = self.snr
330 370 self.dataOut.height = self.height
331 371 self.dataOut.data = self.buffer
332 372 self.dataOut.utctimeInit = self.time
333 373 self.dataOut.utctime = self.dataOut.utctimeInit
334 374 self.dataOut.useLocalTime = False
335 375 self.dataOut.paramInterval = 157
336 376 self.dataOut.timezone = self.timezone
337 377 self.dataOut.site = self.siteFile
338 378 self.dataOut.nrecords = self.nrecords / self.nmodes
339 379 self.dataOut.sizeOfFile = self.sizeOfFile
340 380 self.dataOut.lat = self.lat
341 381 self.dataOut.lon = self.lon
342 382 self.dataOut.channelList = list(range(self.nchannels))
343 383 self.dataOut.kchan = self.kchan
344 384 self.dataOut.delta = self.delta
345 385 self.dataOut.correction = self.correction
346 386 self.dataOut.nmodes = self.nmodes
347 387 self.dataOut.imode = self.imode
348 388 self.dataOut.antenna = self.antenna
349 389 self.dataOut.rx_gains = self.rx_gains
350 390 self.dataOut.flagNoData = False
351 391 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
352 392
353 393 def getData(self):
354 394 '''
355 395 Storing data from databuffer to dataOut object
356 396 '''
357 397 if self.flagNoMoreFiles:
358 398 self.dataOut.flagNoData = True
359 399 self.dataOut.error = 'No More files to read'
400 return
360 401
361 402 if not self.readNextBlock():
362 403 self.dataOut.flagNoData = True
363 return 0
404 self.dataOut.error = 'Time for wait new file reach!!!'
364 405
365 406 self.set_output()
366 407
367 408 return 1
368 409 No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1828 +1,1833
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 25 from schainpy.utils import log
26 26 import schainpy.admin
27 27
28 28 LOCALTIME = True
29 29
30 30
31 31 def isNumber(cad):
32 32 """
33 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
34 34
35 35 Excepciones:
36 36 Si un determinado string no puede ser convertido a numero
37 37 Input:
38 38 str, string al cual se le analiza para determinar si convertible a un numero o no
39 39
40 40 Return:
41 41 True : si el string es uno numerico
42 42 False : no es un string numerico
43 43 """
44 44 try:
45 45 float(cad)
46 46 return True
47 47 except:
48 48 return False
49 49
50 50
51 51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
52 52 """
53 53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
54 54
55 55 Inputs:
56 56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
57 57
58 58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
59 59 segundos contados desde 01/01/1970.
60 60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
61 61 segundos contados desde 01/01/1970.
62 62
63 63 Return:
64 64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
65 65 fecha especificado, de lo contrario retorna False.
66 66
67 67 Excepciones:
68 68 Si el archivo no existe o no puede ser abierto
69 69 Si la cabecera no puede ser leida.
70 70
71 71 """
72 72 basicHeaderObj = BasicHeader(LOCALTIME)
73 73
74 74 try:
75 75 fp = open(filename, 'rb')
76 76 except IOError:
77 77 print("The file %s can't be opened" % (filename))
78 78 return 0
79 79
80 80 sts = basicHeaderObj.read(fp)
81 81 fp.close()
82 82
83 83 if not(sts):
84 84 print("Skipping the file %s because it has not a valid header" % (filename))
85 85 return 0
86 86
87 87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
88 88 return 0
89 89
90 90 return 1
91 91
92 92
93 93 def isTimeInRange(thisTime, startTime, endTime):
94 94 if endTime >= startTime:
95 95 if (thisTime < startTime) or (thisTime > endTime):
96 96 return 0
97 97 return 1
98 98 else:
99 99 if (thisTime < startTime) and (thisTime > endTime):
100 100 return 0
101 101 return 1
102 102
103 103
104 104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
105 105 """
106 106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
107 107
108 108 Inputs:
109 109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
110 110
111 111 startDate : fecha inicial del rango seleccionado en formato datetime.date
112 112
113 113 endDate : fecha final del rango seleccionado en formato datetime.date
114 114
115 115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
116 116
117 117 endTime : tiempo final del rango seleccionado en formato datetime.time
118 118
119 119 Return:
120 120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
121 121 fecha especificado, de lo contrario retorna False.
122 122
123 123 Excepciones:
124 124 Si el archivo no existe o no puede ser abierto
125 125 Si la cabecera no puede ser leida.
126 126
127 127 """
128 128
129 129 try:
130 130 fp = open(filename, 'rb')
131 131 except IOError:
132 132 print("The file %s can't be opened" % (filename))
133 133 return None
134 134
135 135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
136 136 systemHeaderObj = SystemHeader()
137 137 radarControllerHeaderObj = RadarControllerHeader()
138 138 processingHeaderObj = ProcessingHeader()
139 139
140 140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
141 141
142 142 sts = firstBasicHeaderObj.read(fp)
143 143
144 144 if not(sts):
145 145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
146 146 return None
147 147
148 148 if not systemHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not radarControllerHeaderObj.read(fp):
152 152 return None
153 153
154 154 if not processingHeaderObj.read(fp):
155 155 return None
156 156
157 157 filesize = os.path.getsize(filename)
158 158
159 159 offset = processingHeaderObj.blockSize + 24 # header size
160 160
161 161 if filesize <= offset:
162 162 print("[Reading] %s: This file has not enough data" % filename)
163 163 return None
164 164
165 165 fp.seek(-offset, 2)
166 166
167 167 sts = lastBasicHeaderObj.read(fp)
168 168
169 169 fp.close()
170 170
171 171 thisDatetime = lastBasicHeaderObj.datatime
172 172 thisTime_last_block = thisDatetime.time()
173 173
174 174 thisDatetime = firstBasicHeaderObj.datatime
175 175 thisDate = thisDatetime.date()
176 176 thisTime_first_block = thisDatetime.time()
177 177
178 178 # General case
179 179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
180 180 #-----------o----------------------------o-----------
181 181 # startTime endTime
182 182
183 183 if endTime >= startTime:
184 184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
185 185 return None
186 186
187 187 return thisDatetime
188 188
189 189 # If endTime < startTime then endTime belongs to the next day
190 190
191 191 #<<<<<<<<<<<o o>>>>>>>>>>>
192 192 #-----------o----------------------------o-----------
193 193 # endTime startTime
194 194
195 195 if (thisDate == startDate) and (thisTime_last_block < startTime):
196 196 return None
197 197
198 198 if (thisDate == endDate) and (thisTime_first_block > endTime):
199 199 return None
200 200
201 201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
202 202 return None
203 203
204 204 return thisDatetime
205 205
206 206
207 207 def isFolderInDateRange(folder, startDate=None, endDate=None):
208 208 """
209 209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210 210
211 211 Inputs:
212 212 folder : nombre completo del directorio.
213 213 Su formato deberia ser "/path_root/?YYYYDDD"
214 214
215 215 siendo:
216 216 YYYY : Anio (ejemplo 2015)
217 217 DDD : Dia del anio (ejemplo 305)
218 218
219 219 startDate : fecha inicial del rango seleccionado en formato datetime.date
220 220
221 221 endDate : fecha final del rango seleccionado en formato datetime.date
222 222
223 223 Return:
224 224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
225 225 fecha especificado, de lo contrario retorna False.
226 226 Excepciones:
227 227 Si el directorio no tiene el formato adecuado
228 228 """
229 229
230 230 basename = os.path.basename(folder)
231 231
232 232 if not isRadarFolder(basename):
233 233 print("The folder %s has not the rigth format" % folder)
234 234 return 0
235 235
236 236 if startDate and endDate:
237 237 thisDate = getDateFromRadarFolder(basename)
238 238
239 239 if thisDate < startDate:
240 240 return 0
241 241
242 242 if thisDate > endDate:
243 243 return 0
244 244
245 245 return 1
246 246
247 247
248 248 def isFileInDateRange(filename, startDate=None, endDate=None):
249 249 """
250 250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
251 251
252 252 Inputs:
253 253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
254 254
255 255 Su formato deberia ser "?YYYYDDDsss"
256 256
257 257 siendo:
258 258 YYYY : Anio (ejemplo 2015)
259 259 DDD : Dia del anio (ejemplo 305)
260 260 sss : set
261 261
262 262 startDate : fecha inicial del rango seleccionado en formato datetime.date
263 263
264 264 endDate : fecha final del rango seleccionado en formato datetime.date
265 265
266 266 Return:
267 267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
268 268 fecha especificado, de lo contrario retorna False.
269 269 Excepciones:
270 270 Si el archivo no tiene el formato adecuado
271 271 """
272 272
273 273 basename = os.path.basename(filename)
274 274
275 275 if not isRadarFile(basename):
276 276 print("The filename %s has not the rigth format" % filename)
277 277 return 0
278 278
279 279 if startDate and endDate:
280 280 thisDate = getDateFromRadarFile(basename)
281 281
282 282 if thisDate < startDate:
283 283 return 0
284 284
285 285 if thisDate > endDate:
286 286 return 0
287 287
288 288 return 1
289 289
290 290
291 291 def getFileFromSet(path, ext, set):
292 292 validFilelist = []
293 293 fileList = os.listdir(path)
294 294
295 295 # 0 1234 567 89A BCDE
296 296 # H YYYY DDD SSS .ext
297 297
298 298 for thisFile in fileList:
299 299 try:
300 300 year = int(thisFile[1:5])
301 301 doy = int(thisFile[5:8])
302 302 except:
303 303 continue
304 304
305 305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
306 306 continue
307 307
308 308 validFilelist.append(thisFile)
309 309
310 310 myfile = fnmatch.filter(
311 311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
312 312
313 313 if len(myfile) != 0:
314 314 return myfile[0]
315 315 else:
316 316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
317 317 print('the filename %s does not exist' % filename)
318 318 print('...going to the last file: ')
319 319
320 320 if validFilelist:
321 321 validFilelist = sorted(validFilelist, key=str.lower)
322 322 return validFilelist[-1]
323 323
324 324 return None
325 325
326 326
327 327 def getlastFileFromPath(path, ext):
328 328 """
329 329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
330 330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
331 331
332 332 Input:
333 333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
334 334 ext : extension de los files contenidos en una carpeta
335 335
336 336 Return:
337 337 El ultimo file de una determinada carpeta, no se considera el path.
338 338 """
339 339 validFilelist = []
340 340 fileList = os.listdir(path)
341 341
342 342 # 0 1234 567 89A BCDE
343 343 # H YYYY DDD SSS .ext
344 344
345 345 for thisFile in fileList:
346 346
347 347 year = thisFile[1:5]
348 348 if not isNumber(year):
349 349 continue
350 350
351 351 doy = thisFile[5:8]
352 352 if not isNumber(doy):
353 353 continue
354 354
355 355 year = int(year)
356 356 doy = int(doy)
357 357
358 358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
359 359 continue
360 360
361 361 validFilelist.append(thisFile)
362 362
363 363 if validFilelist:
364 364 validFilelist = sorted(validFilelist, key=str.lower)
365 365 return validFilelist[-1]
366 366
367 367 return None
368 368
369 369
370 370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
371 371 """
372 372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
373 373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
374 374 el path exacto de un determinado file.
375 375
376 376 Example :
377 377 nombre correcto del file es .../.../D2009307/P2009307367.ext
378 378
379 379 Entonces la funcion prueba con las siguientes combinaciones
380 380 .../.../y2009307367.ext
381 381 .../.../Y2009307367.ext
382 382 .../.../x2009307/y2009307367.ext
383 383 .../.../x2009307/Y2009307367.ext
384 384 .../.../X2009307/y2009307367.ext
385 385 .../.../X2009307/Y2009307367.ext
386 386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
387 387
388 388 Return:
389 389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
390 390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
391 391 para el filename
392 392 """
393 393 fullfilename = None
394 394 find_flag = False
395 395 filename = None
396 396
397 397 prefixDirList = [None, 'd', 'D']
398 398 if ext.lower() == ".r": # voltage
399 399 prefixFileList = ['d', 'D']
400 400 elif ext.lower() == ".pdata": # spectra
401 401 prefixFileList = ['p', 'P']
402 402 else:
403 403 return None, filename
404 404
405 405 # barrido por las combinaciones posibles
406 406 for prefixDir in prefixDirList:
407 407 thispath = path
408 408 if prefixDir != None:
409 409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
410 410 if foldercounter == 0:
411 411 thispath = os.path.join(path, "%s%04d%03d" %
412 412 (prefixDir, year, doy))
413 413 else:
414 414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
415 415 prefixDir, year, doy, foldercounter))
416 416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
417 417 # formo el nombre del file xYYYYDDDSSS.ext
418 418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
419 419 fullfilename = os.path.join(
420 420 thispath, filename) # formo el path completo
421 421
422 422 if os.path.exists(fullfilename): # verifico que exista
423 423 find_flag = True
424 424 break
425 425 if find_flag:
426 426 break
427 427
428 428 if not(find_flag):
429 429 return None, filename
430 430
431 431 return fullfilename, filename
432 432
433 433
434 434 def isRadarFolder(folder):
435 435 try:
436 436 year = int(folder[1:5])
437 437 doy = int(folder[5:8])
438 438 except:
439 439 return 0
440 440
441 441 return 1
442 442
443 443
444 444 def isRadarFile(file):
445 445 try:
446 446 year = int(file[1:5])
447 447 doy = int(file[5:8])
448 448 set = int(file[8:11])
449 449 except:
450 450 return 0
451 451
452 452 return 1
453 453
454 454
455 455 def getDateFromRadarFile(file):
456 456 try:
457 457 year = int(file[1:5])
458 458 doy = int(file[5:8])
459 459 set = int(file[8:11])
460 460 except:
461 461 return None
462 462
463 463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
464 464 return thisDate
465 465
466 466
467 467 def getDateFromRadarFolder(folder):
468 468 try:
469 469 year = int(folder[1:5])
470 470 doy = int(folder[5:8])
471 471 except:
472 472 return None
473 473
474 474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
475 475 return thisDate
476 476
477 477
478 478 class JRODataIO:
479 479
480 480 c = 3E8
481 481
482 482 isConfig = False
483 483
484 484 basicHeaderObj = None
485 485
486 486 systemHeaderObj = None
487 487
488 488 radarControllerHeaderObj = None
489 489
490 490 processingHeaderObj = None
491 491
492 492 dtype = None
493 493
494 494 pathList = []
495 495
496 496 filenameList = []
497 497
498 498 filename = None
499 499
500 500 ext = None
501 501
502 502 flagIsNewFile = 1
503 503
504 504 flagDiscontinuousBlock = 0
505 505
506 506 flagIsNewBlock = 0
507 507
508 508 fp = None
509 509
510 510 firstHeaderSize = 0
511 511
512 512 basicHeaderSize = 24
513 513
514 514 versionFile = 1103
515 515
516 516 fileSize = None
517 517
518 518 # ippSeconds = None
519 519
520 520 fileSizeByHeader = None
521 521
522 522 fileIndex = None
523 523
524 524 profileIndex = None
525 525
526 526 blockIndex = None
527 527
528 528 nTotalBlocks = None
529 529
530 530 maxTimeStep = 30
531 531
532 532 lastUTTime = None
533 533
534 534 datablock = None
535 535
536 536 dataOut = None
537 537
538 538 blocksize = None
539 539
540 540 getByBlock = False
541 541
542 542 def __init__(self):
543 543
544 544 raise NotImplementedError
545 545
546 546 def run(self):
547 547
548 548 raise NotImplementedError
549 549
550 550 def getDtypeWidth(self):
551 551
552 552 dtype_index = get_dtype_index(self.dtype)
553 553 dtype_width = get_dtype_width(dtype_index)
554 554
555 555 return dtype_width
556 556
557 557 def getAllowedArgs(self):
558 558 if hasattr(self, '__attrs__'):
559 559 return self.__attrs__
560 560 else:
561 561 return inspect.getargspec(self.run).args
562 562
563 563
564 564 class JRODataReader(JRODataIO):
565 565
566 566 online = 0
567 567
568 568 realtime = 0
569 569
570 570 nReadBlocks = 0
571 571
572 572 delay = 10 # number of seconds waiting a new file
573 573
574 574 nTries = 3 # quantity tries
575 575
576 576 nFiles = 3 # number of files for searching
577 577
578 578 path = None
579 579
580 580 foldercounter = 0
581 581
582 582 flagNoMoreFiles = 0
583 583
584 584 datetimeList = []
585 585
586 586 __isFirstTimeOnline = 1
587 587
588 588 __printInfo = True
589 589
590 590 profileIndex = None
591 591
592 592 nTxs = 1
593 593
594 594 txIndex = None
595 595
596 596 # Added--------------------
597 597
598 598 selBlocksize = None
599 599
600 600 selBlocktime = None
601 601
602 602 def __init__(self):
603 603 """
604 604 This class is used to find data files
605 605
606 606 Example:
607 607 reader = JRODataReader()
608 608 fileList = reader.findDataFiles()
609 609
610 610 """
611 611 pass
612 612
613 613 def createObjByDefault(self):
614 614 """
615 615
616 616 """
617 617 raise NotImplementedError
618 618
619 619 def getBlockDimension(self):
620 620
621 621 raise NotImplementedError
622 622
623 623 def searchFilesOffLine(self,
624 624 path,
625 625 startDate=None,
626 626 endDate=None,
627 627 startTime=datetime.time(0, 0, 0),
628 628 endTime=datetime.time(23, 59, 59),
629 629 set=None,
630 630 expLabel='',
631 631 ext='.r',
632 632 cursor=None,
633 633 skip=None,
634 634 walk=True):
635 635
636 636 self.filenameList = []
637 637 self.datetimeList = []
638 638
639 639 pathList = []
640 640
641 641 dateList, pathList = self.findDatafiles(
642 642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
643 643
644 644 if dateList == []:
645 645 return [], []
646 646
647 647 if len(dateList) > 1:
648 648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
649 649 else:
650 650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
651 651
652 652 filenameList = []
653 653 datetimeList = []
654 654
655 655 for thisPath in pathList:
656 656
657 657 fileList = glob.glob1(thisPath, "*%s" % ext)
658 658 fileList.sort()
659 659
660 660 for file in fileList:
661 661
662 662 filename = os.path.join(thisPath, file)
663 663
664 664 if not isFileInDateRange(filename, startDate, endDate):
665 665 continue
666 666
667 667 thisDatetime = isFileInTimeRange(
668 668 filename, startDate, endDate, startTime, endTime)
669 669
670 670 if not(thisDatetime):
671 671 continue
672 672
673 673 filenameList.append(filename)
674 674 datetimeList.append(thisDatetime)
675 675
676 676 if cursor is not None and skip is not None:
677 677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
678 678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
679 679
680 680 if not(filenameList):
681 681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
682 682 return [], []
683 683
684 684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
685 685
686 686 # for i in range(len(filenameList)):
687 687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
688 688
689 689 self.filenameList = filenameList
690 690 self.datetimeList = datetimeList
691 691
692 692 return pathList, filenameList
693 693
694 694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
695 695 """
696 696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
697 697 devuelve el archivo encontrado ademas de otros datos.
698 698
699 699 Input:
700 700 path : carpeta donde estan contenidos los files que contiene data
701 701
702 702 expLabel : Nombre del subexperimento (subfolder)
703 703
704 704 ext : extension de los files
705 705
706 706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
707 707
708 708 Return:
709 709 directory : eL directorio donde esta el file encontrado
710 710 filename : el ultimo file de una determinada carpeta
711 711 year : el anho
712 712 doy : el numero de dia del anho
713 713 set : el set del archivo
714 714
715 715
716 716 """
717 717 if not os.path.isdir(path):
718 718 return None, None, None, None, None, None
719 719
720 720 dirList = []
721 721
722 722 if not walk:
723 723 fullpath = path
724 724 foldercounter = 0
725 725 else:
726 726 # Filtra solo los directorios
727 727 for thisPath in os.listdir(path):
728 728 if not os.path.isdir(os.path.join(path, thisPath)):
729 729 continue
730 730 if not isRadarFolder(thisPath):
731 731 continue
732 732
733 733 dirList.append(thisPath)
734 734
735 735 if not(dirList):
736 736 return None, None, None, None, None, None
737 737
738 738 dirList = sorted(dirList, key=str.lower)
739 739
740 740 doypath = dirList[-1]
741 741 foldercounter = int(doypath.split('_')[1]) if len(
742 742 doypath.split('_')) > 1 else 0
743 743 fullpath = os.path.join(path, doypath, expLabel)
744 744
745 745 print("[Reading] %s folder was found: " % (fullpath))
746 746
747 747 if set == None:
748 748 filename = getlastFileFromPath(fullpath, ext)
749 749 else:
750 750 filename = getFileFromSet(fullpath, ext, set)
751 751
752 752 if not(filename):
753 753 return None, None, None, None, None, None
754 754
755 755 print("[Reading] %s file was found" % (filename))
756 756
757 757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
758 758 return None, None, None, None, None, None
759 759
760 760 year = int(filename[1:5])
761 761 doy = int(filename[5:8])
762 762 set = int(filename[8:11])
763 763
764 764 return fullpath, foldercounter, filename, year, doy, set
765 765
766 766 def __setNextFileOffline(self):
767 767
768 768 idFile = self.fileIndex
769 769
770 770 while (True):
771 771 idFile += 1
772 772 if not(idFile < len(self.filenameList)):
773 773 self.flagNoMoreFiles = 1
774 774 # print "[Reading] No more Files"
775 775 return 0
776 776
777 777 filename = self.filenameList[idFile]
778 778
779 779 if not(self.__verifyFile(filename)):
780 780 continue
781 781
782 782 fileSize = os.path.getsize(filename)
783 783 fp = open(filename, 'rb')
784 784 break
785 785
786 786 self.flagIsNewFile = 1
787 787 self.fileIndex = idFile
788 788 self.filename = filename
789 789 self.fileSize = fileSize
790 790 self.fp = fp
791 791
792 792 # print "[Reading] Setting the file: %s"%self.filename
793 793
794 794 return 1
795 795
796 796 def __setNextFileOnline(self):
797 797 """
798 798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
799 799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
800 800 siguientes.
801 801
802 802 Affected:
803 803 self.flagIsNewFile
804 804 self.filename
805 805 self.fileSize
806 806 self.fp
807 807 self.set
808 808 self.flagNoMoreFiles
809 809
810 810 Return:
811 811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
812 812 1 : si el file fue abierto con exito y esta listo a ser leido
813 813
814 814 Excepciones:
815 815 Si un determinado file no puede ser abierto
816 816 """
817 817 nFiles = 0
818 818 fileOk_flag = False
819 819 firstTime_flag = True
820 820
821 821 self.set += 1
822 822
823 823 if self.set > 999:
824 824 self.set = 0
825 825 self.foldercounter += 1
826 826
827 827 # busca el 1er file disponible
828 828 fullfilename, filename = checkForRealPath(
829 829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
830 830 if fullfilename:
831 831 if self.__verifyFile(fullfilename, False):
832 832 fileOk_flag = True
833 833
834 834 # si no encuentra un file entonces espera y vuelve a buscar
835 835 if not(fileOk_flag):
836 836 # busco en los siguientes self.nFiles+1 files posibles
837 837 for nFiles in range(self.nFiles + 1):
838 838
839 839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
840 840 tries = self.nTries
841 841 else:
842 842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
843 843
844 844 for nTries in range(tries):
845 845 if firstTime_flag:
846 846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
847 847 sleep(self.delay)
848 848 else:
849 849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
850 850
851 851 fullfilename, filename = checkForRealPath(
852 852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
853 853 if fullfilename:
854 854 if self.__verifyFile(fullfilename):
855 855 fileOk_flag = True
856 856 break
857 857
858 858 if fileOk_flag:
859 859 break
860 860
861 861 firstTime_flag = False
862 862
863 863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
864 864 self.set += 1
865 865
866 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
867 867 if nFiles == (self.nFiles - 1):
868 868 self.set = 0
869 869 self.doy += 1
870 870 self.foldercounter = 0
871 871
872 872 if fileOk_flag:
873 873 self.fileSize = os.path.getsize(fullfilename)
874 874 self.filename = fullfilename
875 875 self.flagIsNewFile = 1
876 876 if self.fp != None:
877 877 self.fp.close()
878 878 self.fp = open(fullfilename, 'rb')
879 879 self.flagNoMoreFiles = 0
880 880 # print '[Reading] Setting the file: %s' % fullfilename
881 881 else:
882 882 self.fileSize = 0
883 883 self.filename = None
884 884 self.flagIsNewFile = 0
885 885 self.fp = None
886 886 self.flagNoMoreFiles = 1
887 887 # print '[Reading] No more files to read'
888 888
889 889 return fileOk_flag
890 890
891 891 def setNextFile(self):
892 892 if self.fp != None:
893 893 self.fp.close()
894 894
895 895 if self.online:
896 896 newFile = self.__setNextFileOnline()
897 897 else:
898 898 newFile = self.__setNextFileOffline()
899 899
900 900 if not(newFile):
901 901 self.dataOut.error = 'No more files to read'
902 902 return 0
903 903
904 904 if self.verbose:
905 905 print('[Reading] Setting the file: %s' % self.filename)
906 906
907 907 self.__readFirstHeader()
908 908 self.nReadBlocks = 0
909 909 return 1
910 910
911 911 def __waitNewBlock(self):
912 912 """
913 913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914 914
915 915 Si el modo de lectura es OffLine siempre retorn 0
916 916 """
917 917 if not self.online:
918 918 return 0
919 919
920 920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 921 return 0
922 922
923 923 currentPointer = self.fp.tell()
924 924
925 925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926 926
927 927 for nTries in range(self.nTries):
928 928
929 929 self.fp.close()
930 930 self.fp = open(self.filename, 'rb')
931 931 self.fp.seek(currentPointer)
932 932
933 933 self.fileSize = os.path.getsize(self.filename)
934 934 currentSize = self.fileSize - currentPointer
935 935
936 936 if (currentSize >= neededSize):
937 937 self.basicHeaderObj.read(self.fp)
938 938 return 1
939 939
940 940 if self.fileSize == self.fileSizeByHeader:
941 941 # self.flagEoF = True
942 942 return 0
943 943
944 944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 945 sleep(self.delay)
946 946
947 947 return 0
948 948
949 def waitDataBlock(self, pointer_location):
949 def waitDataBlock(self, pointer_location, blocksize=None):
950 950
951 951 currentPointer = pointer_location
952
953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
952 if blocksize is None:
953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 else:
955 neededSize = blocksize
954 956
955 957 for nTries in range(self.nTries):
956 958 self.fp.close()
957 959 self.fp = open(self.filename, 'rb')
958 960 self.fp.seek(currentPointer)
959 961
960 962 self.fileSize = os.path.getsize(self.filename)
961 963 currentSize = self.fileSize - currentPointer
962 964
963 965 if (currentSize >= neededSize):
964 966 return 1
965 967
966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
968 log.warning(
969 "Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1),
970 self.name
971 )
967 972 sleep(self.delay)
968 973
969 974 return 0
970 975
971 976 def __jumpToLastBlock(self):
972 977
973 978 if not(self.__isFirstTimeOnline):
974 979 return
975 980
976 981 csize = self.fileSize - self.fp.tell()
977 982 blocksize = self.processingHeaderObj.blockSize
978 983
979 984 # salta el primer bloque de datos
980 985 if csize > self.processingHeaderObj.blockSize:
981 986 self.fp.seek(self.fp.tell() + blocksize)
982 987 else:
983 988 return
984 989
985 990 csize = self.fileSize - self.fp.tell()
986 991 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
987 992 while True:
988 993
989 994 if self.fp.tell() < self.fileSize:
990 995 self.fp.seek(self.fp.tell() + neededsize)
991 996 else:
992 997 self.fp.seek(self.fp.tell() - neededsize)
993 998 break
994 999
995 1000 # csize = self.fileSize - self.fp.tell()
996 1001 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
997 1002 # factor = int(csize/neededsize)
998 1003 # if factor > 0:
999 1004 # self.fp.seek(self.fp.tell() + factor*neededsize)
1000 1005
1001 1006 self.flagIsNewFile = 0
1002 1007 self.__isFirstTimeOnline = 0
1003 1008
1004 1009 def __setNewBlock(self):
1005 1010 # if self.server is None:
1006 1011 if self.fp == None:
1007 1012 return 0
1008 1013
1009 1014 # if self.online:
1010 1015 # self.__jumpToLastBlock()
1011 1016
1012 1017 if self.flagIsNewFile:
1013 1018 self.lastUTTime = self.basicHeaderObj.utc
1014 1019 return 1
1015 1020
1016 1021 if self.realtime:
1017 1022 self.flagDiscontinuousBlock = 1
1018 1023 if not(self.setNextFile()):
1019 1024 return 0
1020 1025 else:
1021 1026 return 1
1022 1027 # if self.server is None:
1023 1028 currentSize = self.fileSize - self.fp.tell()
1024 1029 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1025 1030 if (currentSize >= neededSize):
1026 1031 self.basicHeaderObj.read(self.fp)
1027 1032 self.lastUTTime = self.basicHeaderObj.utc
1028 1033 return 1
1029 1034 # else:
1030 1035 # self.basicHeaderObj.read(self.zHeader)
1031 1036 # self.lastUTTime = self.basicHeaderObj.utc
1032 1037 # return 1
1033 1038 if self.__waitNewBlock():
1034 1039 self.lastUTTime = self.basicHeaderObj.utc
1035 1040 return 1
1036 1041 # if self.server is None:
1037 1042 if not(self.setNextFile()):
1038 1043 return 0
1039 1044
1040 1045 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1041 1046 self.lastUTTime = self.basicHeaderObj.utc
1042 1047
1043 1048 self.flagDiscontinuousBlock = 0
1044 1049
1045 1050 if deltaTime > self.maxTimeStep:
1046 1051 self.flagDiscontinuousBlock = 1
1047 1052
1048 1053 return 1
1049 1054
1050 1055 def readNextBlock(self):
1051 1056
1052 1057 # Skip block out of startTime and endTime
1053 1058 while True:
1054 1059 if not(self.__setNewBlock()):
1055 1060 self.dataOut.error = 'No more files to read'
1056 1061 return 0
1057 1062
1058 1063 if not(self.readBlock()):
1059 1064 return 0
1060 1065
1061 1066 self.getBasicHeader()
1062 1067 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1063 1068 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1064 1069 self.processingHeaderObj.dataBlocksPerFile,
1065 1070 self.dataOut.datatime.ctime()))
1066 1071 continue
1067 1072
1068 1073 break
1069 1074
1070 1075 if self.verbose:
1071 1076 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1072 1077 self.processingHeaderObj.dataBlocksPerFile,
1073 1078 self.dataOut.datatime.ctime()))
1074 1079 return 1
1075 1080
1076 1081 def __readFirstHeader(self):
1077 1082
1078 1083 self.basicHeaderObj.read(self.fp)
1079 1084 self.systemHeaderObj.read(self.fp)
1080 1085 self.radarControllerHeaderObj.read(self.fp)
1081 1086 self.processingHeaderObj.read(self.fp)
1082 1087
1083 1088 self.firstHeaderSize = self.basicHeaderObj.size
1084 1089
1085 1090 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1086 1091 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1087 1092 if datatype == 0:
1088 1093 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1089 1094 elif datatype == 1:
1090 1095 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1091 1096 elif datatype == 2:
1092 1097 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1093 1098 elif datatype == 3:
1094 1099 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1095 1100 elif datatype == 4:
1096 1101 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1097 1102 elif datatype == 5:
1098 1103 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1099 1104 else:
1100 1105 raise ValueError('Data type was not defined')
1101 1106
1102 1107 self.dtype = datatype_str
1103 1108 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1104 1109 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1105 1110 self.firstHeaderSize + self.basicHeaderSize * \
1106 1111 (self.processingHeaderObj.dataBlocksPerFile - 1)
1107 1112 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1108 1113 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1109 1114 self.getBlockDimension()
1110 1115
1111 1116 def __verifyFile(self, filename, msgFlag=True):
1112 1117
1113 1118 msg = None
1114 1119
1115 1120 try:
1116 1121 fp = open(filename, 'rb')
1117 1122 except IOError:
1118 1123
1119 1124 if msgFlag:
1120 1125 print("[Reading] File %s can't be opened" % (filename))
1121 1126
1122 1127 return False
1123 1128
1124 1129 currentPosition = fp.tell()
1125 1130 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1126 1131
1127 1132 if neededSize == 0:
1128 1133 basicHeaderObj = BasicHeader(LOCALTIME)
1129 1134 systemHeaderObj = SystemHeader()
1130 1135 radarControllerHeaderObj = RadarControllerHeader()
1131 1136 processingHeaderObj = ProcessingHeader()
1132 1137
1133 1138 if not(basicHeaderObj.read(fp)):
1134 1139 fp.close()
1135 1140 return False
1136 1141
1137 1142 if not(systemHeaderObj.read(fp)):
1138 1143 fp.close()
1139 1144 return False
1140 1145
1141 1146 if not(radarControllerHeaderObj.read(fp)):
1142 1147 fp.close()
1143 1148 return False
1144 1149
1145 1150 if not(processingHeaderObj.read(fp)):
1146 1151 fp.close()
1147 1152 return False
1148 1153
1149 1154 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1150 1155 else:
1151 1156 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1152 1157
1153 1158 fp.close()
1154 1159
1155 1160 fileSize = os.path.getsize(filename)
1156 1161 currentSize = fileSize - currentPosition
1157 1162
1158 1163 if currentSize < neededSize:
1159 1164 if msgFlag and (msg != None):
1160 1165 print(msg)
1161 1166 return False
1162 1167
1163 1168 return True
1164 1169
1165 1170 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1166 1171
1167 1172 path_empty = True
1168 1173
1169 1174 dateList = []
1170 1175 pathList = []
1171 1176
1172 1177 multi_path = path.split(',')
1173 1178
1174 1179 if not walk:
1175 1180
1176 1181 for single_path in multi_path:
1177 1182
1178 1183 if not os.path.isdir(single_path):
1179 1184 continue
1180 1185
1181 1186 fileList = glob.glob1(single_path, "*" + ext)
1182 1187
1183 1188 if not fileList:
1184 1189 continue
1185 1190
1186 1191 path_empty = False
1187 1192
1188 1193 fileList.sort()
1189 1194
1190 1195 for thisFile in fileList:
1191 1196
1192 1197 if not os.path.isfile(os.path.join(single_path, thisFile)):
1193 1198 continue
1194 1199
1195 1200 if not isRadarFile(thisFile):
1196 1201 continue
1197 1202
1198 1203 if not isFileInDateRange(thisFile, startDate, endDate):
1199 1204 continue
1200 1205
1201 1206 thisDate = getDateFromRadarFile(thisFile)
1202 1207
1203 1208 if thisDate in dateList:
1204 1209 continue
1205 1210
1206 1211 dateList.append(thisDate)
1207 1212 pathList.append(single_path)
1208 1213
1209 1214 else:
1210 1215 for single_path in multi_path:
1211 1216
1212 1217 if not os.path.isdir(single_path):
1213 1218 continue
1214 1219
1215 1220 dirList = []
1216 1221
1217 1222 for thisPath in os.listdir(single_path):
1218 1223
1219 1224 if not os.path.isdir(os.path.join(single_path, thisPath)):
1220 1225 continue
1221 1226
1222 1227 if not isRadarFolder(thisPath):
1223 1228 continue
1224 1229
1225 1230 if not isFolderInDateRange(thisPath, startDate, endDate):
1226 1231 continue
1227 1232
1228 1233 dirList.append(thisPath)
1229 1234
1230 1235 if not dirList:
1231 1236 continue
1232 1237
1233 1238 dirList.sort()
1234 1239
1235 1240 for thisDir in dirList:
1236 1241
1237 1242 datapath = os.path.join(single_path, thisDir, expLabel)
1238 1243 fileList = glob.glob1(datapath, "*" + ext)
1239 1244
1240 1245 if not fileList:
1241 1246 continue
1242 1247
1243 1248 path_empty = False
1244 1249
1245 1250 thisDate = getDateFromRadarFolder(thisDir)
1246 1251
1247 1252 pathList.append(datapath)
1248 1253 dateList.append(thisDate)
1249 1254
1250 1255 dateList.sort()
1251 1256
1252 1257 if walk:
1253 1258 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1254 1259 else:
1255 1260 pattern_path = multi_path[0]
1256 1261
1257 1262 if path_empty:
1258 1263 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1259 1264 else:
1260 1265 if not dateList:
1261 1266 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1262 1267
1263 1268 if include_path:
1264 1269 return dateList, pathList
1265 1270
1266 1271 return dateList
1267 1272
1268 1273 def setup(self,
1269 1274 path=None,
1270 1275 startDate=None,
1271 1276 endDate=None,
1272 1277 startTime=datetime.time(0, 0, 0),
1273 1278 endTime=datetime.time(23, 59, 59),
1274 1279 set=None,
1275 1280 expLabel="",
1276 1281 ext=None,
1277 1282 online=False,
1278 1283 delay=60,
1279 1284 walk=True,
1280 1285 getblock=False,
1281 1286 nTxs=1,
1282 1287 realtime=False,
1283 1288 blocksize=None,
1284 1289 blocktime=None,
1285 1290 skip=None,
1286 1291 cursor=None,
1287 1292 warnings=True,
1288 1293 verbose=True,
1289 1294 server=None,
1290 1295 format=None,
1291 1296 oneDDict=None,
1292 1297 twoDDict=None,
1293 1298 independentParam=None):
1294 1299 if server is not None:
1295 1300 if 'tcp://' in server:
1296 1301 address = server
1297 1302 else:
1298 1303 address = 'ipc:///tmp/%s' % server
1299 1304 self.server = address
1300 1305 self.context = zmq.Context()
1301 1306 self.receiver = self.context.socket(zmq.PULL)
1302 1307 self.receiver.connect(self.server)
1303 1308 time.sleep(0.5)
1304 1309 print('[Starting] ReceiverData from {}'.format(self.server))
1305 1310 else:
1306 1311 self.server = None
1307 1312 if path == None:
1308 1313 raise ValueError("[Reading] The path is not valid")
1309 1314
1310 1315 if ext == None:
1311 1316 ext = self.ext
1312 1317
1313 1318 if online:
1314 1319 print("[Reading] Searching files in online mode...")
1315 1320
1316 1321 for nTries in range(self.nTries):
1317 1322 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1318 1323 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1319 1324
1320 1325 if fullpath:
1321 1326 break
1322 1327
1323 1328 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1324 1329 sleep(self.delay)
1325 1330
1326 1331 if not(fullpath):
1327 1332 self.dataOut.error = 'There isn\'t any valid file in {}'.format(path)
1328 1333 return
1329 1334
1330 1335 self.year = year
1331 1336 self.doy = doy
1332 1337 self.set = set - 1
1333 1338 self.path = path
1334 1339 self.foldercounter = foldercounter
1335 1340 last_set = None
1336 1341 else:
1337 1342 print("[Reading] Searching files in offline mode ...")
1338 1343 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1339 1344 startTime=startTime, endTime=endTime,
1340 1345 set=set, expLabel=expLabel, ext=ext,
1341 1346 walk=walk, cursor=cursor,
1342 1347 skip=skip)
1343 1348
1344 1349 if not(pathList):
1345 1350 self.fileIndex = -1
1346 1351 self.pathList = []
1347 1352 self.filenameList = []
1348 1353 return
1349 1354
1350 1355 self.fileIndex = -1
1351 1356 self.pathList = pathList
1352 1357 self.filenameList = filenameList
1353 1358 file_name = os.path.basename(filenameList[-1])
1354 1359 basename, ext = os.path.splitext(file_name)
1355 1360 last_set = int(basename[-3:])
1356 1361
1357 1362 self.online = online
1358 1363 self.realtime = realtime
1359 1364 self.delay = delay
1360 1365 ext = ext.lower()
1361 1366 self.ext = ext
1362 1367 self.getByBlock = getblock
1363 1368 self.nTxs = nTxs
1364 1369 self.startTime = startTime
1365 1370 self.endTime = endTime
1366 1371 self.endDate = endDate
1367 1372 self.startDate = startDate
1368 1373 # Added-----------------
1369 1374 self.selBlocksize = blocksize
1370 1375 self.selBlocktime = blocktime
1371 1376
1372 1377 # Verbose-----------
1373 1378 self.verbose = verbose
1374 1379 self.warnings = warnings
1375 1380
1376 1381 if not(self.setNextFile()):
1377 1382 if (startDate != None) and (endDate != None):
1378 1383 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1379 1384 elif startDate != None:
1380 1385 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1381 1386 else:
1382 1387 print("[Reading] No files")
1383 1388
1384 1389 self.fileIndex = -1
1385 1390 self.pathList = []
1386 1391 self.filenameList = []
1387 1392 return
1388 1393
1389 1394 # self.getBasicHeader()
1390 1395
1391 1396 if last_set != None:
1392 1397 self.dataOut.last_block = last_set * \
1393 1398 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1394 1399 return
1395 1400
1396 1401 def getBasicHeader(self):
1397 1402
1398 1403 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1399 1404 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1400 1405
1401 1406 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1402 1407
1403 1408 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1404 1409
1405 1410 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1406 1411
1407 1412 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1408 1413
1409 1414 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1410 1415
1411 1416 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1412 1417
1413 1418 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1414 1419
1415 1420 def getFirstHeader(self):
1416 1421
1417 1422 raise NotImplementedError
1418 1423
1419 1424 def getData(self):
1420 1425
1421 1426 raise NotImplementedError
1422 1427
1423 1428 def hasNotDataInBuffer(self):
1424 1429
1425 1430 raise NotImplementedError
1426 1431
1427 1432 def readBlock(self):
1428 1433
1429 1434 raise NotImplementedError
1430 1435
1431 1436 def isEndProcess(self):
1432 1437
1433 1438 return self.flagNoMoreFiles
1434 1439
1435 1440 def printReadBlocks(self):
1436 1441
1437 1442 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1438 1443
1439 1444 def printTotalBlocks(self):
1440 1445
1441 1446 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1442 1447
1443 1448 def printNumberOfBlock(self):
1444 1449 'SPAM!'
1445 1450
1446 1451 # if self.flagIsNewBlock:
1447 1452 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1448 1453 # self.processingHeaderObj.dataBlocksPerFile,
1449 1454 # self.dataOut.datatime.ctime())
1450 1455
1451 1456 def printInfo(self):
1452 1457
1453 1458 if self.__printInfo == False:
1454 1459 return
1455 1460
1456 1461 self.basicHeaderObj.printInfo()
1457 1462 self.systemHeaderObj.printInfo()
1458 1463 self.radarControllerHeaderObj.printInfo()
1459 1464 self.processingHeaderObj.printInfo()
1460 1465
1461 1466 self.__printInfo = False
1462 1467
1463 1468 def run(self,
1464 1469 path=None,
1465 1470 startDate=None,
1466 1471 endDate=None,
1467 1472 startTime=datetime.time(0, 0, 0),
1468 1473 endTime=datetime.time(23, 59, 59),
1469 1474 set=None,
1470 1475 expLabel="",
1471 1476 ext=None,
1472 1477 online=False,
1473 1478 delay=60,
1474 1479 walk=True,
1475 1480 getblock=False,
1476 1481 nTxs=1,
1477 1482 realtime=False,
1478 1483 blocksize=None,
1479 1484 blocktime=None,
1480 1485 skip=None,
1481 1486 cursor=None,
1482 1487 warnings=True,
1483 1488 server=None,
1484 1489 verbose=True,
1485 1490 format=None,
1486 1491 oneDDict=None,
1487 1492 twoDDict=None,
1488 1493 independentParam=None, **kwargs):
1489 1494
1490 1495 if not(self.isConfig):
1491 1496 self.setup(path=path,
1492 1497 startDate=startDate,
1493 1498 endDate=endDate,
1494 1499 startTime=startTime,
1495 1500 endTime=endTime,
1496 1501 set=set,
1497 1502 expLabel=expLabel,
1498 1503 ext=ext,
1499 1504 online=online,
1500 1505 delay=delay,
1501 1506 walk=walk,
1502 1507 getblock=getblock,
1503 1508 nTxs=nTxs,
1504 1509 realtime=realtime,
1505 1510 blocksize=blocksize,
1506 1511 blocktime=blocktime,
1507 1512 skip=skip,
1508 1513 cursor=cursor,
1509 1514 warnings=warnings,
1510 1515 server=server,
1511 1516 verbose=verbose,
1512 1517 format=format,
1513 1518 oneDDict=oneDDict,
1514 1519 twoDDict=twoDDict,
1515 1520 independentParam=independentParam)
1516 1521 self.isConfig = True
1517 1522 if server is None:
1518 1523 self.getData()
1519 1524 else:
1520 1525 self.getFromServer()
1521 1526
1522 1527
1523 1528 class JRODataWriter(JRODataIO):
1524 1529
1525 1530 """
1526 1531 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1527 1532 de los datos siempre se realiza por bloques.
1528 1533 """
1529 1534
1530 1535 blockIndex = 0
1531 1536
1532 1537 path = None
1533 1538
1534 1539 setFile = None
1535 1540
1536 1541 profilesPerBlock = None
1537 1542
1538 1543 blocksPerFile = None
1539 1544
1540 1545 nWriteBlocks = 0
1541 1546
1542 1547 fileDate = None
1543 1548
1544 1549 def __init__(self, dataOut=None):
1545 1550 raise NotImplementedError
1546 1551
1547 1552 def hasAllDataInBuffer(self):
1548 1553 raise NotImplementedError
1549 1554
1550 1555 def setBlockDimension(self):
1551 1556 raise NotImplementedError
1552 1557
1553 1558 def writeBlock(self):
1554 1559 raise NotImplementedError
1555 1560
1556 1561 def putData(self):
1557 1562 raise NotImplementedError
1558 1563
1559 1564 def getProcessFlags(self):
1560 1565
1561 1566 processFlags = 0
1562 1567
1563 1568 dtype_index = get_dtype_index(self.dtype)
1564 1569 procflag_dtype = get_procflag_dtype(dtype_index)
1565 1570
1566 1571 processFlags += procflag_dtype
1567 1572
1568 1573 if self.dataOut.flagDecodeData:
1569 1574 processFlags += PROCFLAG.DECODE_DATA
1570 1575
1571 1576 if self.dataOut.flagDeflipData:
1572 1577 processFlags += PROCFLAG.DEFLIP_DATA
1573 1578
1574 1579 if self.dataOut.code is not None:
1575 1580 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1576 1581
1577 1582 if self.dataOut.nCohInt > 1:
1578 1583 processFlags += PROCFLAG.COHERENT_INTEGRATION
1579 1584
1580 1585 if self.dataOut.type == "Spectra":
1581 1586 if self.dataOut.nIncohInt > 1:
1582 1587 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1583 1588
1584 1589 if self.dataOut.data_dc is not None:
1585 1590 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1586 1591
1587 1592 if self.dataOut.flagShiftFFT:
1588 1593 processFlags += PROCFLAG.SHIFT_FFT_DATA
1589 1594
1590 1595 return processFlags
1591 1596
1592 1597 def setBasicHeader(self):
1593 1598
1594 1599 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1595 1600 self.basicHeaderObj.version = self.versionFile
1596 1601 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1597 1602
1598 1603 utc = numpy.floor(self.dataOut.utctime)
1599 1604 milisecond = (self.dataOut.utctime - utc) * 1000.0
1600 1605
1601 1606 self.basicHeaderObj.utc = utc
1602 1607 self.basicHeaderObj.miliSecond = milisecond
1603 1608 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1604 1609 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1605 1610 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1606 1611
1607 1612 def setFirstHeader(self):
1608 1613 """
1609 1614 Obtiene una copia del First Header
1610 1615
1611 1616 Affected:
1612 1617
1613 1618 self.basicHeaderObj
1614 1619 self.systemHeaderObj
1615 1620 self.radarControllerHeaderObj
1616 1621 self.processingHeaderObj self.
1617 1622
1618 1623 Return:
1619 1624 None
1620 1625 """
1621 1626
1622 1627 raise NotImplementedError
1623 1628
1624 1629 def __writeFirstHeader(self):
1625 1630 """
1626 1631 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1627 1632
1628 1633 Affected:
1629 1634 __dataType
1630 1635
1631 1636 Return:
1632 1637 None
1633 1638 """
1634 1639
1635 1640 # CALCULAR PARAMETROS
1636 1641
1637 1642 sizeLongHeader = self.systemHeaderObj.size + \
1638 1643 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1639 1644 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1640 1645
1641 1646 self.basicHeaderObj.write(self.fp)
1642 1647 self.systemHeaderObj.write(self.fp)
1643 1648 self.radarControllerHeaderObj.write(self.fp)
1644 1649 self.processingHeaderObj.write(self.fp)
1645 1650
1646 1651 def __setNewBlock(self):
1647 1652 """
1648 1653 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1649 1654
1650 1655 Return:
1651 1656 0 : si no pudo escribir nada
1652 1657 1 : Si escribio el Basic el First Header
1653 1658 """
1654 1659 if self.fp == None:
1655 1660 self.setNextFile()
1656 1661
1657 1662 if self.flagIsNewFile:
1658 1663 return 1
1659 1664
1660 1665 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1661 1666 self.basicHeaderObj.write(self.fp)
1662 1667 return 1
1663 1668
1664 1669 if not(self.setNextFile()):
1665 1670 return 0
1666 1671
1667 1672 return 1
1668 1673
1669 1674 def writeNextBlock(self):
1670 1675 """
1671 1676 Selecciona el bloque siguiente de datos y los escribe en un file
1672 1677
1673 1678 Return:
1674 1679 0 : Si no hizo pudo escribir el bloque de datos
1675 1680 1 : Si no pudo escribir el bloque de datos
1676 1681 """
1677 1682 if not(self.__setNewBlock()):
1678 1683 return 0
1679 1684
1680 1685 self.writeBlock()
1681 1686
1682 1687 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1683 1688 self.processingHeaderObj.dataBlocksPerFile))
1684 1689
1685 1690 return 1
1686 1691
1687 1692 def setNextFile(self):
1688 1693 """
1689 1694 Determina el siguiente file que sera escrito
1690 1695
1691 1696 Affected:
1692 1697 self.filename
1693 1698 self.subfolder
1694 1699 self.fp
1695 1700 self.setFile
1696 1701 self.flagIsNewFile
1697 1702
1698 1703 Return:
1699 1704 0 : Si el archivo no puede ser escrito
1700 1705 1 : Si el archivo esta listo para ser escrito
1701 1706 """
1702 1707 ext = self.ext
1703 1708 path = self.path
1704 1709
1705 1710 if self.fp != None:
1706 1711 self.fp.close()
1707 1712
1708 1713 timeTuple = time.localtime(self.dataOut.utctime)
1709 1714 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1710 1715
1711 1716 fullpath = os.path.join(path, subfolder)
1712 1717 setFile = self.setFile
1713 1718
1714 1719 if not(os.path.exists(fullpath)):
1715 1720 os.mkdir(fullpath)
1716 1721 setFile = -1 # inicializo mi contador de seteo
1717 1722 else:
1718 1723 filesList = os.listdir(fullpath)
1719 1724 if len(filesList) > 0:
1720 1725 filesList = sorted(filesList, key=str.lower)
1721 1726 filen = filesList[-1]
1722 1727 # el filename debera tener el siguiente formato
1723 1728 # 0 1234 567 89A BCDE (hex)
1724 1729 # x YYYY DDD SSS .ext
1725 1730 if isNumber(filen[8:11]):
1726 1731 # inicializo mi contador de seteo al seteo del ultimo file
1727 1732 setFile = int(filen[8:11])
1728 1733 else:
1729 1734 setFile = -1
1730 1735 else:
1731 1736 setFile = -1 # inicializo mi contador de seteo
1732 1737
1733 1738 setFile += 1
1734 1739
1735 1740 # If this is a new day it resets some values
1736 1741 if self.dataOut.datatime.date() > self.fileDate:
1737 1742 setFile = 0
1738 1743 self.nTotalBlocks = 0
1739 1744
1740 1745 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1741 1746 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1742 1747
1743 1748 filename = os.path.join(path, subfolder, filen)
1744 1749
1745 1750 fp = open(filename, 'wb')
1746 1751
1747 1752 self.blockIndex = 0
1748 1753
1749 1754 # guardando atributos
1750 1755 self.filename = filename
1751 1756 self.subfolder = subfolder
1752 1757 self.fp = fp
1753 1758 self.setFile = setFile
1754 1759 self.flagIsNewFile = 1
1755 1760 self.fileDate = self.dataOut.datatime.date()
1756 1761
1757 1762 self.setFirstHeader()
1758 1763
1759 1764 print('[Writing] Opening file: %s' % self.filename)
1760 1765
1761 1766 self.__writeFirstHeader()
1762 1767
1763 1768 return 1
1764 1769
1765 1770 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1766 1771 """
1767 1772 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1768 1773
1769 1774 Inputs:
1770 1775 path : directory where data will be saved
1771 1776 profilesPerBlock : number of profiles per block
1772 1777 set : initial file set
1773 1778 datatype : An integer number that defines data type:
1774 1779 0 : int8 (1 byte)
1775 1780 1 : int16 (2 bytes)
1776 1781 2 : int32 (4 bytes)
1777 1782 3 : int64 (8 bytes)
1778 1783 4 : float32 (4 bytes)
1779 1784 5 : double64 (8 bytes)
1780 1785
1781 1786 Return:
1782 1787 0 : Si no realizo un buen seteo
1783 1788 1 : Si realizo un buen seteo
1784 1789 """
1785 1790
1786 1791 if ext == None:
1787 1792 ext = self.ext
1788 1793
1789 1794 self.ext = ext.lower()
1790 1795
1791 1796 self.path = path
1792 1797
1793 1798 if set is None:
1794 1799 self.setFile = -1
1795 1800 else:
1796 1801 self.setFile = set - 1
1797 1802
1798 1803 self.blocksPerFile = blocksPerFile
1799 1804
1800 1805 self.profilesPerBlock = profilesPerBlock
1801 1806
1802 1807 self.dataOut = dataOut
1803 1808 self.fileDate = self.dataOut.datatime.date()
1804 1809 # By default
1805 1810 self.dtype = self.dataOut.dtype
1806 1811
1807 1812 if datatype is not None:
1808 1813 self.dtype = get_numpy_dtype(datatype)
1809 1814
1810 1815 if not(self.setNextFile()):
1811 1816 print("[Writing] There isn't a next file")
1812 1817 return 0
1813 1818
1814 1819 self.setBlockDimension()
1815 1820
1816 1821 return 1
1817 1822
1818 1823 def run(self, dataOut, path, blocksPerFile=100, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1819 1824
1820 1825 if not(self.isConfig):
1821 1826
1822 1827 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1823 1828 set=set, ext=ext, datatype=datatype, **kwargs)
1824 1829 self.isConfig = True
1825 1830
1826 1831 self.dataOut = dataOut
1827 1832 self.putData()
1828 1833 return self.dataOut No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now