##// END OF EJS Templates
schain
RSS
little formatting
José Chávez -
r1013:4d72229ee73f
parent child
Show More
Side by Side Unified
Context file:
r1013:4d72229ee73f Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,240 +1,240
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os, sys
7 7 import datetime
8 8 import numpy
9 9 import traceback
10 10
11 11 from time import sleep
12 12 from threading import Lock
13 13 # from threading import Thread
14 14
15 15 import schainpy
16 16 import schainpy.admin
17 17
18 18 from schainpy.model.proc.jroproc_base import Operation
19 19 from schainpy.model.serializer.data import obj2Dict, dict2Obj
20 20 from jroplot_correlation import *
21 21 from jroplot_heispectra import *
22 22 from jroplot_parameters import *
23 23 from jroplot_spectra import *
24 24 from jroplot_voltage import *
25 25
26 26
27 27 class Plotter(Operation):
28 28
29 29 isConfig = None
30 30 name = None
31 31 __queue = None
32 32
33 33 def __init__(self, plotter_name, plotter_queue=None):
34 34
35 35 Operation.__init__(self)
36 36
37 37 self.isConfig = False
38 38 self.name = plotter_name
39 39 self.__queue = plotter_queue
40 40
41 41 def getSubplots(self):
42 42
43 43 nrow = self.nplots
44 44 ncol = 1
45 45 return nrow, ncol
46 46
47 47 def setup(self, **kwargs):
48 48
49 49 print "Initializing ..."
50 50
51 51
52 52 def run(self, dataOut, id=None, **kwargs):
53 53
54 54 """
55 55
56 56 Input:
57 57 dataOut :
58 58 id :
59 59 """
60 60
61 61 packDict = {}
62 62
63 63 packDict['id'] = id
64 64 packDict['name'] = self.name
65 65 packDict['kwargs'] = kwargs
66 66
67 67 # packDict['data'] = obj2Dict(dataOut)
68 68 packDict['data'] = dataOut
69 69
70 70 self.__queue.put(packDict)
71 71
72 72 # class PlotManager(Thread):
73 73 class PlotManager():
74 74
75 75 __err = False
76 76 __stop = False
77 77 __realtime = False
78 78
79 79 controllerThreadObj = None
80 80
81 81 plotterList = ['Scope',
82 82 'SpectraPlot', 'RTIPlot',
83 83 'SpectraCutPlot',
84 84 'CrossSpectraPlot', 'CoherenceMap',
85 85 'PowerProfilePlot', 'Noise', 'BeaconPhase',
86 86 'CorrelationPlot',
87 87 'SpectraHeisScope','RTIfromSpectraHeis']
88 88
89 89 def __init__(self, plotter_queue):
90 90
91 91 # Thread.__init__(self)
92 92 # self.setDaemon(True)
93 93
94 94 self.__queue = plotter_queue
95 95 self.__lock = Lock()
96 96
97 97 self.plotInstanceDict = {}
98 98
99 99 self.__err = False
100 100 self.__stop = False
101 101 self.__realtime = False
102 102
103 103 def __handleError(self, name="", send_email=False):
104 104
105 105 err = traceback.format_exception(sys.exc_info()[0],
106 106 sys.exc_info()[1],
107 107 sys.exc_info()[2])
108 108
109 109 print "***** Error occurred in PlotManager *****"
110 110 print "***** [%s]: %s" %(name, err[-1])
111 111
112 112 message = "\nError ocurred in %s:\n" %name
113 113 message += "".join(err)
114 114
115 115 sys.stderr.write(message)
116 116
117 117 if not send_email:
118 118 return
119 119
120 120 import socket
121 121
122 122 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, name)
123 123
124 124 subtitle = "%s:\n" %(name)
125 125 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
126 126 subtitle += "Working directory: %s\n" %os.path.abspath("./")
127 # subtitle += "Configuration file: %s\n" %self.filename
127 # subtitle += "Configuration file: %s\n" %self.filename
128 128 subtitle += "Time: %s\n" %str(datetime.datetime.now())
129 129
130 130 adminObj = schainpy.admin.SchainNotify()
131 131 adminObj.sendAlert(message=message,
132 132 subject=subject,
133 133 subtitle=subtitle)
134 134
135 135 def run(self):
136 136
137 137 if self.__queue.empty():
138 138 return
139 139
140 140 if self.__err:
141 141 serial_data = self.__queue.get()
142 142 self.__queue.task_done()
143 143 return
144 144
145 145 self.__lock.acquire()
146 146
147 147 # if self.__queue.full():
148 148 # for i in range(int(self.__queue.qsize()/2)):
149 149 # serial_data = self.__queue.get()
150 150 # self.__queue.task_done()
151 151
152 152 n = int(self.__queue.qsize()/3 + 1)
153 153
154 154 for i in range(n):
155 155
156 156 if self.__queue.empty():
157 157 break
158 158
159 159 serial_data = self.__queue.get()
160 160 self.__queue.task_done()
161 161
162 162 plot_id = serial_data['id']
163 163 plot_name = serial_data['name']
164 164 kwargs = serial_data['kwargs']
165 165 # dataDict = serial_data['data']
166 166 #
167 167 # dataPlot = dict2Obj(dataDict)
168 168
169 169 dataPlot = serial_data['data']
170 170
171 171 if plot_id not in self.plotInstanceDict.keys():
172 172 className = eval(plot_name)
173 173 self.plotInstanceDict[plot_id] = className()
174 174
175 175 plotter = self.plotInstanceDict[plot_id]
176 176 try:
177 177 plotter.run(dataPlot, plot_id, **kwargs)
178 178 except:
179 179 self.__err = True
180 180 self.__handleError(plot_name, send_email=True)
181 181 break
182 182
183 183 self.__lock.release()
184 184
185 185 def isEmpty(self):
186 186
187 187 return self.__queue.empty()
188 188
189 189 def stop(self):
190 190
191 191 self.__lock.acquire()
192 192
193 193 self.__stop = True
194 194
195 195 self.__lock.release()
196 196
197 197 def close(self):
198 198
199 199 self.__lock.acquire()
200 200
201 201 for plot_id in self.plotInstanceDict.keys():
202 202 plotter = self.plotInstanceDict[plot_id]
203 203 plotter.close()
204 204
205 205 self.__lock.release()
206 206
207 207 def setController(self, controllerThreadObj):
208 208
209 209 self.controllerThreadObj = controllerThreadObj
210 210
211 211 def start(self):
212 212
213 213 if not self.controllerThreadObj.isRunning():
214 214 raise RuntimeError, "controllerThreadObj has not been initialized. Use controllerThreadObj.start() before call this method"
215 215
216 216 self.join()
217 217
218 218 def join(self):
219 219
220 220 #Execute plotter while controller is running
221 221 while self.controllerThreadObj.isRunning():
222 222 self.run()
223 223
224 224 self.controllerThreadObj.stop()
225 225
226 226 #Wait until plotter queue is empty
227 227 while not self.isEmpty():
228 228 self.run()
229 229
230 230 self.close()
231 231
232 232 def isErrorDetected(self):
233 233
234 234 self.__lock.acquire()
235 235
236 236 err = self.__err
237 237
238 238 self.__lock.release()
239 239
240 240 return err No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1723 +1,1723
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 print "The file %s can't be opened" %(filename)
72 72 return 0
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isTimeInRange(thisTime, startTime, endTime):
87 87
88 88 if endTime >= startTime:
89 89 if (thisTime < startTime) or (thisTime > endTime):
90 90 return 0
91 91
92 92 return 1
93 93 else:
94 94 if (thisTime < startTime) and (thisTime > endTime):
95 95 return 0
96 96
97 97 return 1
98 98
99 99 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
100 100 """
101 101 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
102 102
103 103 Inputs:
104 104 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
105 105
106 106 startDate : fecha inicial del rango seleccionado en formato datetime.date
107 107
108 108 endDate : fecha final del rango seleccionado en formato datetime.date
109 109
110 110 startTime : tiempo inicial del rango seleccionado en formato datetime.time
111 111
112 112 endTime : tiempo final del rango seleccionado en formato datetime.time
113 113
114 114 Return:
115 115 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
116 116 fecha especificado, de lo contrario retorna False.
117 117
118 118 Excepciones:
119 119 Si el archivo no existe o no puede ser abierto
120 120 Si la cabecera no puede ser leida.
121 121
122 122 """
123 123
124 124
125 125 try:
126 126 fp = open(filename,'rb')
127 127 except IOError:
128 128 print "The file %s can't be opened" %(filename)
129 129 return None
130 130
131 131 firstBasicHeaderObj = BasicHeader(LOCALTIME)
132 132 systemHeaderObj = SystemHeader()
133 133 radarControllerHeaderObj = RadarControllerHeader()
134 134 processingHeaderObj = ProcessingHeader()
135 135
136 136 lastBasicHeaderObj = BasicHeader(LOCALTIME)
137 137
138 138 sts = firstBasicHeaderObj.read(fp)
139 139
140 140 if not(sts):
141 141 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
142 142 return None
143 143
144 144 if not systemHeaderObj.read(fp):
145 145 return None
146 146
147 147 if not radarControllerHeaderObj.read(fp):
148 148 return None
149 149
150 150 if not processingHeaderObj.read(fp):
151 151 return None
152 152
153 153 filesize = os.path.getsize(filename)
154 154
155 155 offset = processingHeaderObj.blockSize + 24 #header size
156 156
157 157 if filesize <= offset:
158 158 print "[Reading] %s: This file has not enough data" %filename
159 159 return None
160 160
161 161 fp.seek(-offset, 2)
162 162
163 163 sts = lastBasicHeaderObj.read(fp)
164 164
165 165 fp.close()
166 166
167 167 thisDatetime = lastBasicHeaderObj.datatime
168 168 thisTime_last_block = thisDatetime.time()
169 169
170 170 thisDatetime = firstBasicHeaderObj.datatime
171 171 thisDate = thisDatetime.date()
172 172 thisTime_first_block = thisDatetime.time()
173 173
174 174 #General case
175 175 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
176 176 #-----------o----------------------------o-----------
177 177 # startTime endTime
178 178
179 179 if endTime >= startTime:
180 180 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
181 181 return None
182 182
183 183 return thisDatetime
184 184
185 185 #If endTime < startTime then endTime belongs to the next day
186 186
187 187
188 188 #<<<<<<<<<<<o o>>>>>>>>>>>
189 189 #-----------o----------------------------o-----------
190 190 # endTime startTime
191 191
192 192 if (thisDate == startDate) and (thisTime_last_block < startTime):
193 193 return None
194 194
195 195 if (thisDate == endDate) and (thisTime_first_block > endTime):
196 196 return None
197 197
198 198 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
199 199 return None
200 200
201 201 return thisDatetime
202 202
203 203 def isFolderInDateRange(folder, startDate=None, endDate=None):
204 204 """
205 205 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
206 206
207 207 Inputs:
208 208 folder : nombre completo del directorio.
209 209 Su formato deberia ser "/path_root/?YYYYDDD"
210 210
211 211 siendo:
212 212 YYYY : Anio (ejemplo 2015)
213 213 DDD : Dia del anio (ejemplo 305)
214 214
215 215 startDate : fecha inicial del rango seleccionado en formato datetime.date
216 216
217 217 endDate : fecha final del rango seleccionado en formato datetime.date
218 218
219 219 Return:
220 220 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
221 221 fecha especificado, de lo contrario retorna False.
222 222 Excepciones:
223 223 Si el directorio no tiene el formato adecuado
224 224 """
225 225
226 226 basename = os.path.basename(folder)
227 227
228 228 if not isRadarFolder(basename):
229 229 print "The folder %s has not the rigth format" %folder
230 230 return 0
231 231
232 232 if startDate and endDate:
233 233 thisDate = getDateFromRadarFolder(basename)
234 234
235 235 if thisDate < startDate:
236 236 return 0
237 237
238 238 if thisDate > endDate:
239 239 return 0
240 240
241 241 return 1
242 242
243 243 def isFileInDateRange(filename, startDate=None, endDate=None):
244 244 """
245 245 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
246 246
247 247 Inputs:
248 248 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
249 249
250 250 Su formato deberia ser "?YYYYDDDsss"
251 251
252 252 siendo:
253 253 YYYY : Anio (ejemplo 2015)
254 254 DDD : Dia del anio (ejemplo 305)
255 255 sss : set
256 256
257 257 startDate : fecha inicial del rango seleccionado en formato datetime.date
258 258
259 259 endDate : fecha final del rango seleccionado en formato datetime.date
260 260
261 261 Return:
262 262 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
263 263 fecha especificado, de lo contrario retorna False.
264 264 Excepciones:
265 265 Si el archivo no tiene el formato adecuado
266 266 """
267 267
268 268 basename = os.path.basename(filename)
269 269
270 270 if not isRadarFile(basename):
271 271 print "The filename %s has not the rigth format" %filename
272 272 return 0
273 273
274 274 if startDate and endDate:
275 275 thisDate = getDateFromRadarFile(basename)
276 276
277 277 if thisDate < startDate:
278 278 return 0
279 279
280 280 if thisDate > endDate:
281 281 return 0
282 282
283 283 return 1
284 284
285 285 def getFileFromSet(path, ext, set):
286 286 validFilelist = []
287 287 fileList = os.listdir(path)
288 288
289 289 # 0 1234 567 89A BCDE
290 290 # H YYYY DDD SSS .ext
291 291
292 292 for thisFile in fileList:
293 293 try:
294 294 year = int(thisFile[1:5])
295 295 doy = int(thisFile[5:8])
296 296 except:
297 297 continue
298 298
299 299 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
300 300 continue
301 301
302 302 validFilelist.append(thisFile)
303 303
304 304 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
305 305
306 306 if len(myfile)!= 0:
307 307 return myfile[0]
308 308 else:
309 309 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
310 310 print 'the filename %s does not exist'%filename
311 311 print '...going to the last file: '
312 312
313 313 if validFilelist:
314 314 validFilelist = sorted( validFilelist, key=str.lower )
315 315 return validFilelist[-1]
316 316
317 317 return None
318 318
319 319 def getlastFileFromPath(path, ext):
320 320 """
321 321 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
322 322 al final de la depuracion devuelve el ultimo file de la lista que quedo.
323 323
324 324 Input:
325 325 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
326 326 ext : extension de los files contenidos en una carpeta
327 327
328 328 Return:
329 329 El ultimo file de una determinada carpeta, no se considera el path.
330 330 """
331 331 validFilelist = []
332 332 fileList = os.listdir(path)
333 333
334 334 # 0 1234 567 89A BCDE
335 335 # H YYYY DDD SSS .ext
336 336
337 337 for thisFile in fileList:
338 338
339 339 year = thisFile[1:5]
340 340 if not isNumber(year):
341 341 continue
342 342
343 343 doy = thisFile[5:8]
344 344 if not isNumber(doy):
345 345 continue
346 346
347 347 year = int(year)
348 348 doy = int(doy)
349 349
350 350 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
351 351 continue
352 352
353 353 validFilelist.append(thisFile)
354 354
355 355 if validFilelist:
356 356 validFilelist = sorted( validFilelist, key=str.lower )
357 357 return validFilelist[-1]
358 358
359 359 return None
360 360
361 361 def checkForRealPath(path, foldercounter, year, doy, set, ext):
362 362 """
363 363 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
364 364 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
365 365 el path exacto de un determinado file.
366 366
367 367 Example :
368 368 nombre correcto del file es .../.../D2009307/P2009307367.ext
369 369
370 370 Entonces la funcion prueba con las siguientes combinaciones
371 371 .../.../y2009307367.ext
372 372 .../.../Y2009307367.ext
373 373 .../.../x2009307/y2009307367.ext
374 374 .../.../x2009307/Y2009307367.ext
375 375 .../.../X2009307/y2009307367.ext
376 376 .../.../X2009307/Y2009307367.ext
377 377 siendo para este caso, la ultima combinacion de letras, identica al file buscado
378 378
379 379 Return:
380 380 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
381 381 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
382 382 para el filename
383 383 """
384 384 fullfilename = None
385 385 find_flag = False
386 386 filename = None
387 387
388 388 prefixDirList = [None,'d','D']
389 389 if ext.lower() == ".r": #voltage
390 390 prefixFileList = ['d','D']
391 391 elif ext.lower() == ".pdata": #spectra
392 392 prefixFileList = ['p','P']
393 393 else:
394 394 return None, filename
395 395
396 396 #barrido por las combinaciones posibles
397 397 for prefixDir in prefixDirList:
398 398 thispath = path
399 399 if prefixDir != None:
400 400 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
401 401 if foldercounter == 0:
402 402 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
403 403 else:
404 404 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
405 405 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
406 406 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
407 407 fullfilename = os.path.join( thispath, filename ) #formo el path completo
408 408
409 409 if os.path.exists( fullfilename ): #verifico que exista
410 410 find_flag = True
411 411 break
412 412 if find_flag:
413 413 break
414 414
415 415 if not(find_flag):
416 416 return None, filename
417 417
418 418 return fullfilename, filename
419 419
420 420 def isRadarFolder(folder):
421 421 try:
422 422 year = int(folder[1:5])
423 423 doy = int(folder[5:8])
424 424 except:
425 425 return 0
426 426
427 427 return 1
428 428
429 429 def isRadarFile(file):
430 430 try:
431 431 year = int(file[1:5])
432 432 doy = int(file[5:8])
433 433 set = int(file[8:11])
434 434 except:
435 435 return 0
436 436
437 437 return 1
438 438
439 439 def getDateFromRadarFile(file):
440 440 try:
441 441 year = int(file[1:5])
442 442 doy = int(file[5:8])
443 443 set = int(file[8:11])
444 444 except:
445 445 return None
446 446
447 447 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
448 448 return thisDate
449 449
450 450 def getDateFromRadarFolder(folder):
451 451 try:
452 452 year = int(folder[1:5])
453 453 doy = int(folder[5:8])
454 454 except:
455 455 return None
456 456
457 457 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
458 458 return thisDate
459 459
460 460 class JRODataIO:
461 461
462 462 c = 3E8
463 463
464 464 isConfig = False
465 465
466 466 basicHeaderObj = None
467 467
468 468 systemHeaderObj = None
469 469
470 470 radarControllerHeaderObj = None
471 471
472 472 processingHeaderObj = None
473 473
474 474 dtype = None
475 475
476 476 pathList = []
477 477
478 478 filenameList = []
479 479
480 480 filename = None
481 481
482 482 ext = None
483 483
484 484 flagIsNewFile = 1
485 485
486 486 flagDiscontinuousBlock = 0
487 487
488 488 flagIsNewBlock = 0
489 489
490 490 fp = None
491 491
492 492 firstHeaderSize = 0
493 493
494 494 basicHeaderSize = 24
495 495
496 496 versionFile = 1103
497 497
498 498 fileSize = None
499 499
500 500 # ippSeconds = None
501 501
502 502 fileSizeByHeader = None
503 503
504 504 fileIndex = None
505 505
506 506 profileIndex = None
507 507
508 508 blockIndex = None
509 509
510 510 nTotalBlocks = None
511 511
512 512 maxTimeStep = 30
513 513
514 514 lastUTTime = None
515 515
516 516 datablock = None
517 517
518 518 dataOut = None
519 519
520 520 blocksize = None
521 521
522 522 getByBlock = False
523 523
524 524 def __init__(self):
525 525
526 526 raise NotImplementedError
527 527
528 528 def run(self):
529 529
530 530 raise NotImplementedError
531 531
532 532 def getDtypeWidth(self):
533 533
534 534 dtype_index = get_dtype_index(self.dtype)
535 535 dtype_width = get_dtype_width(dtype_index)
536 536
537 537 return dtype_width
538 538
539 539 class JRODataReader(JRODataIO):
540 540
541 541
542 542 online = 0
543 543
544 544 realtime = 0
545 545
546 546 nReadBlocks = 0
547 547
548 548 delay = 10 #number of seconds waiting a new file
549 549
550 550 nTries = 3 #quantity tries
551 551
552 552 nFiles = 3 #number of files for searching
553 553
554 554 path = None
555 555
556 556 foldercounter = 0
557 557
558 558 flagNoMoreFiles = 0
559 559
560 560 datetimeList = []
561 561
562 562 __isFirstTimeOnline = 1
563 563
564 564 __printInfo = True
565 565
566 566 profileIndex = None
567 567
568 568 nTxs = 1
569 569
570 570 txIndex = None
571 571
572 572 #Added--------------------
573 573
574 574 selBlocksize = None
575 575
576 576 selBlocktime = None
577 577
578 578
579 579 def __init__(self):
580 580
581 581 """
582 582 This class is used to find data files
583 583
584 584 Example:
585 585 reader = JRODataReader()
586 586 fileList = reader.findDataFiles()
587 587
588 588 """
589 589 pass
590 590
591 591
592 592 def createObjByDefault(self):
593 593 """
594 594
595 595 """
596 596 raise NotImplementedError
597 597
598 598 def getBlockDimension(self):
599 599
600 600 raise NotImplementedError
601 601
602 602 def __searchFilesOffLine(self,
603 603 path,
604 604 startDate=None,
605 605 endDate=None,
606 606 startTime=datetime.time(0,0,0),
607 607 endTime=datetime.time(23,59,59),
608 608 set=None,
609 609 expLabel='',
610 610 ext='.r',
611 611 walk=True):
612 612
613 613 self.filenameList = []
614 614 self.datetimeList = []
615 615
616 616 pathList = []
617 617
618 618 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
619 619
620 620 if dateList == []:
621 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
621 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
622 622 return None, None
623 623
624 624 if len(dateList) > 1:
625 625 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
626 626 else:
627 627 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
628 628
629 629 filenameList = []
630 630 datetimeList = []
631 631
632 632 for thisPath in pathList:
633 # thisPath = pathList[pathDict[file]]
633 # thisPath = pathList[pathDict[file]]
634 634
635 635 fileList = glob.glob1(thisPath, "*%s" %ext)
636 636 fileList.sort()
637 637
638 638 for file in fileList:
639 639
640 640 filename = os.path.join(thisPath,file)
641 641
642 642 if not isFileInDateRange(filename, startDate, endDate):
643 643 continue
644 644
645 645 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
646 646
647 647 if not(thisDatetime):
648 648 continue
649 649
650 650 filenameList.append(filename)
651 651 datetimeList.append(thisDatetime)
652 652
653 653 if not(filenameList):
654 654 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
655 655 return None, None
656 656
657 657 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
658 658 print
659 659
660 660 for i in range(len(filenameList)):
661 661 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
662 662
663 663 self.filenameList = filenameList
664 664 self.datetimeList = datetimeList
665 665
666 666 return pathList, filenameList
667 667
668 668 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
669 669
670 670 """
671 671 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
672 672 devuelve el archivo encontrado ademas de otros datos.
673 673
674 674 Input:
675 675 path : carpeta donde estan contenidos los files que contiene data
676 676
677 677 expLabel : Nombre del subexperimento (subfolder)
678 678
679 679 ext : extension de los files
680 680
681 681 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
682 682
683 683 Return:
684 684 directory : eL directorio donde esta el file encontrado
685 685 filename : el ultimo file de una determinada carpeta
686 686 year : el anho
687 687 doy : el numero de dia del anho
688 688 set : el set del archivo
689 689
690 690
691 691 """
692 692 if not os.path.isdir(path):
693 693 return None, None, None, None, None, None
694 694
695 695 dirList = []
696 696
697 697 if not walk:
698 698 fullpath = path
699 699 foldercounter = 0
700 700 else:
701 701 #Filtra solo los directorios
702 702 for thisPath in os.listdir(path):
703 703 if not os.path.isdir(os.path.join(path,thisPath)):
704 704 continue
705 705 if not isRadarFolder(thisPath):
706 706 continue
707 707
708 708 dirList.append(thisPath)
709 709
710 710 if not(dirList):
711 711 return None, None, None, None, None, None
712 712
713 713 dirList = sorted( dirList, key=str.lower )
714 714
715 715 doypath = dirList[-1]
716 716 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
717 717 fullpath = os.path.join(path, doypath, expLabel)
718 718
719 719
720 720 print "[Reading] %s folder was found: " %(fullpath )
721 721
722 722 if set == None:
723 723 filename = getlastFileFromPath(fullpath, ext)
724 724 else:
725 725 filename = getFileFromSet(fullpath, ext, set)
726 726
727 727 if not(filename):
728 728 return None, None, None, None, None, None
729 729
730 730 print "[Reading] %s file was found" %(filename)
731 731
732 732 if not(self.__verifyFile(os.path.join(fullpath, filename))):
733 733 return None, None, None, None, None, None
734 734
735 735 year = int( filename[1:5] )
736 736 doy = int( filename[5:8] )
737 737 set = int( filename[8:11] )
738 738
739 739 return fullpath, foldercounter, filename, year, doy, set
740 740
741 741 def __setNextFileOffline(self):
742 742
743 743 idFile = self.fileIndex
744 744
745 745 while (True):
746 746 idFile += 1
747 747 if not(idFile < len(self.filenameList)):
748 748 self.flagNoMoreFiles = 1
749 # print "[Reading] No more Files"
749 # print "[Reading] No more Files"
750 750 return 0
751 751
752 752 filename = self.filenameList[idFile]
753 753
754 754 if not(self.__verifyFile(filename)):
755 755 continue
756 756
757 757 fileSize = os.path.getsize(filename)
758 758 fp = open(filename,'rb')
759 759 break
760 760
761 761 self.flagIsNewFile = 1
762 762 self.fileIndex = idFile
763 763 self.filename = filename
764 764 self.fileSize = fileSize
765 765 self.fp = fp
766 766
767 # print "[Reading] Setting the file: %s"%self.filename
767 # print "[Reading] Setting the file: %s"%self.filename
768 768
769 769 return 1
770 770
771 771 def __setNextFileOnline(self):
772 772 """
773 773 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
774 774 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
775 775 siguientes.
776 776
777 777 Affected:
778 778 self.flagIsNewFile
779 779 self.filename
780 780 self.fileSize
781 781 self.fp
782 782 self.set
783 783 self.flagNoMoreFiles
784 784
785 785 Return:
786 786 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
787 787 1 : si el file fue abierto con exito y esta listo a ser leido
788 788
789 789 Excepciones:
790 790 Si un determinado file no puede ser abierto
791 791 """
792 792 nFiles = 0
793 793 fileOk_flag = False
794 794 firstTime_flag = True
795 795
796 796 self.set += 1
797 797
798 798 if self.set > 999:
799 799 self.set = 0
800 800 self.foldercounter += 1
801 801
802 802 #busca el 1er file disponible
803 803 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
804 804 if fullfilename:
805 805 if self.__verifyFile(fullfilename, False):
806 806 fileOk_flag = True
807 807
808 808 #si no encuentra un file entonces espera y vuelve a buscar
809 809 if not(fileOk_flag):
810 810 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
811 811
812 812 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
813 813 tries = self.nTries
814 814 else:
815 815 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
816 816
817 817 for nTries in range( tries ):
818 818 if firstTime_flag:
819 819 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
820 820 sleep( self.delay )
821 821 else:
822 822 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
823 823
824 824 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
825 825 if fullfilename:
826 826 if self.__verifyFile(fullfilename):
827 827 fileOk_flag = True
828 828 break
829 829
830 830 if fileOk_flag:
831 831 break
832 832
833 833 firstTime_flag = False
834 834
835 835 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
836 836 self.set += 1
837 837
838 838 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
839 839 self.set = 0
840 840 self.doy += 1
841 841 self.foldercounter = 0
842 842
843 843 if fileOk_flag:
844 844 self.fileSize = os.path.getsize( fullfilename )
845 845 self.filename = fullfilename
846 846 self.flagIsNewFile = 1
847 847 if self.fp != None: self.fp.close()
848 848 self.fp = open(fullfilename, 'rb')
849 849 self.flagNoMoreFiles = 0
850 # print '[Reading] Setting the file: %s' % fullfilename
850 # print '[Reading] Setting the file: %s' % fullfilename
851 851 else:
852 852 self.fileSize = 0
853 853 self.filename = None
854 854 self.flagIsNewFile = 0
855 855 self.fp = None
856 856 self.flagNoMoreFiles = 1
857 # print '[Reading] No more files to read'
857 # print '[Reading] No more files to read'
858 858
859 859 return fileOk_flag
860 860
861 861 def setNextFile(self):
862 862 if self.fp != None:
863 863 self.fp.close()
864 864
865 865 if self.online:
866 866 newFile = self.__setNextFileOnline()
867 867 else:
868 868 newFile = self.__setNextFileOffline()
869 869
870 870 if not(newFile):
871 871 print '[Reading] No more files to read'
872 872 return 0
873 873
874 874 print '[Reading] Setting the file: %s' % self.filename
875 875
876 876 self.__readFirstHeader()
877 877 self.nReadBlocks = 0
878 878 return 1
879 879
880 880 def __waitNewBlock(self):
881 881 """
882 882 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
883 883
884 884 Si el modo de lectura es OffLine siempre retorn 0
885 885 """
886 886 if not self.online:
887 887 return 0
888 888
889 889 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
890 890 return 0
891 891
892 892 currentPointer = self.fp.tell()
893 893
894 894 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
895 895
896 896 for nTries in range( self.nTries ):
897 897
898 898 self.fp.close()
899 899 self.fp = open( self.filename, 'rb' )
900 900 self.fp.seek( currentPointer )
901 901
902 902 self.fileSize = os.path.getsize( self.filename )
903 903 currentSize = self.fileSize - currentPointer
904 904
905 905 if ( currentSize >= neededSize ):
906 906 self.basicHeaderObj.read(self.fp)
907 907 return 1
908 908
909 909 if self.fileSize == self.fileSizeByHeader:
910 # self.flagEoF = True
910 # self.flagEoF = True
911 911 return 0
912 912
913 913 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
914 914 sleep( self.delay )
915 915
916 916
917 917 return 0
918 918
919 919 def waitDataBlock(self,pointer_location):
920 920
921 921 currentPointer = pointer_location
922 922
923 923 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
924 924
925 925 for nTries in range( self.nTries ):
926 926 self.fp.close()
927 927 self.fp = open( self.filename, 'rb' )
928 928 self.fp.seek( currentPointer )
929 929
930 930 self.fileSize = os.path.getsize( self.filename )
931 931 currentSize = self.fileSize - currentPointer
932 932
933 933 if ( currentSize >= neededSize ):
934 934 return 1
935 935
936 936 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
937 937 sleep( self.delay )
938 938
939 939 return 0
940 940
941 941 def __jumpToLastBlock(self):
942 942
943 943 if not(self.__isFirstTimeOnline):
944 944 return
945 945
946 946 csize = self.fileSize - self.fp.tell()
947 947 blocksize = self.processingHeaderObj.blockSize
948 948
949 949 #salta el primer bloque de datos
950 950 if csize > self.processingHeaderObj.blockSize:
951 951 self.fp.seek(self.fp.tell() + blocksize)
952 952 else:
953 953 return
954 954
955 955 csize = self.fileSize - self.fp.tell()
956 956 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
957 957 while True:
958 958
959 959 if self.fp.tell()<self.fileSize:
960 960 self.fp.seek(self.fp.tell() + neededsize)
961 961 else:
962 962 self.fp.seek(self.fp.tell() - neededsize)
963 963 break
964 964
965 # csize = self.fileSize - self.fp.tell()
966 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
967 # factor = int(csize/neededsize)
968 # if factor > 0:
969 # self.fp.seek(self.fp.tell() + factor*neededsize)
965 # csize = self.fileSize - self.fp.tell()
966 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
967 # factor = int(csize/neededsize)
968 # if factor > 0:
969 # self.fp.seek(self.fp.tell() + factor*neededsize)
970 970
971 971 self.flagIsNewFile = 0
972 972 self.__isFirstTimeOnline = 0
973 973
974 974 def __setNewBlock(self):
975 975
976 976 if self.fp == None:
977 977 return 0
978 978
979 # if self.online:
980 # self.__jumpToLastBlock()
979 # if self.online:
980 # self.__jumpToLastBlock()
981 981
982 982 if self.flagIsNewFile:
983 983 self.lastUTTime = self.basicHeaderObj.utc
984 984 return 1
985 985
986 986 if self.realtime:
987 987 self.flagDiscontinuousBlock = 1
988 988 if not(self.setNextFile()):
989 989 return 0
990 990 else:
991 991 return 1
992 992
993 993 currentSize = self.fileSize - self.fp.tell()
994 994 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995 995
996 996 if (currentSize >= neededSize):
997 997 self.basicHeaderObj.read(self.fp)
998 998 self.lastUTTime = self.basicHeaderObj.utc
999 999 return 1
1000 1000
1001 1001 if self.__waitNewBlock():
1002 1002 self.lastUTTime = self.basicHeaderObj.utc
1003 1003 return 1
1004 1004
1005 1005 if not(self.setNextFile()):
1006 1006 return 0
1007 1007
1008 1008 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1009 1009 self.lastUTTime = self.basicHeaderObj.utc
1010 1010
1011 1011 self.flagDiscontinuousBlock = 0
1012 1012
1013 1013 if deltaTime > self.maxTimeStep:
1014 1014 self.flagDiscontinuousBlock = 1
1015 1015
1016 1016 return 1
1017 1017
1018 1018 def readNextBlock(self):
1019 1019
1020 1020 #Skip block out of startTime and endTime
1021 1021 while True:
1022 1022 if not(self.__setNewBlock()):
1023 1023 return 0
1024 1024
1025 1025 if not(self.readBlock()):
1026 1026 return 0
1027 1027
1028 1028 self.getBasicHeader()
1029 1029
1030 1030 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1031 1031
1032 1032 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1033 1033 self.processingHeaderObj.dataBlocksPerFile,
1034 1034 self.dataOut.datatime.ctime())
1035 1035 continue
1036 1036
1037 1037 break
1038 1038
1039 1039 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1040 1040 self.processingHeaderObj.dataBlocksPerFile,
1041 1041 self.dataOut.datatime.ctime())
1042 1042 return 1
1043 1043
1044 1044 def __readFirstHeader(self):
1045 1045
1046 1046 self.basicHeaderObj.read(self.fp)
1047 1047 self.systemHeaderObj.read(self.fp)
1048 1048 self.radarControllerHeaderObj.read(self.fp)
1049 1049 self.processingHeaderObj.read(self.fp)
1050 1050
1051 1051 self.firstHeaderSize = self.basicHeaderObj.size
1052 1052
1053 1053 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1054 1054 if datatype == 0:
1055 1055 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1056 1056 elif datatype == 1:
1057 1057 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1058 1058 elif datatype == 2:
1059 1059 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1060 1060 elif datatype == 3:
1061 1061 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1062 1062 elif datatype == 4:
1063 1063 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1064 1064 elif datatype == 5:
1065 1065 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1066 1066 else:
1067 1067 raise ValueError, 'Data type was not defined'
1068 1068
1069 1069 self.dtype = datatype_str
1070 1070 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1071 1071 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1072 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1073 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1072 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1073 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1074 1074 self.getBlockDimension()
1075 1075
1076 1076 def __verifyFile(self, filename, msgFlag=True):
1077 1077
1078 1078 msg = None
1079 1079
1080 1080 try:
1081 1081 fp = open(filename, 'rb')
1082 1082 except IOError:
1083 1083
1084 1084 if msgFlag:
1085 1085 print "[Reading] File %s can't be opened" % (filename)
1086 1086
1087 1087 return False
1088 1088
1089 1089 currentPosition = fp.tell()
1090 1090 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1091 1091
1092 1092 if neededSize == 0:
1093 1093 basicHeaderObj = BasicHeader(LOCALTIME)
1094 1094 systemHeaderObj = SystemHeader()
1095 1095 radarControllerHeaderObj = RadarControllerHeader()
1096 1096 processingHeaderObj = ProcessingHeader()
1097 1097
1098 1098 if not( basicHeaderObj.read(fp) ):
1099 1099 fp.close()
1100 1100 return False
1101 1101
1102 1102 if not( systemHeaderObj.read(fp) ):
1103 1103 fp.close()
1104 1104 return False
1105 1105
1106 1106 if not( radarControllerHeaderObj.read(fp) ):
1107 1107 fp.close()
1108 1108 return False
1109 1109
1110 1110 if not( processingHeaderObj.read(fp) ):
1111 1111 fp.close()
1112 1112 return False
1113 1113
1114 1114 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1115 1115 else:
1116 1116 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1117 1117
1118 1118 fp.close()
1119 1119
1120 1120 fileSize = os.path.getsize(filename)
1121 1121 currentSize = fileSize - currentPosition
1122 1122
1123 1123 if currentSize < neededSize:
1124 1124 if msgFlag and (msg != None):
1125 1125 print msg
1126 1126 return False
1127 1127
1128 1128 return True
1129 1129
1130 1130 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1131 1131
1132 1132 path_empty = True
1133 1133
1134 1134 dateList = []
1135 1135 pathList = []
1136 1136
1137 1137 multi_path = path.split(',')
1138 1138
1139 1139 if not walk:
1140 1140
1141 1141 for single_path in multi_path:
1142 1142
1143 1143 if not os.path.isdir(single_path):
1144 1144 continue
1145 1145
1146 1146 fileList = glob.glob1(single_path, "*"+ext)
1147 1147
1148 1148 if not fileList:
1149 1149 continue
1150 1150
1151 1151 path_empty = False
1152 1152
1153 1153 fileList.sort()
1154 1154
1155 1155 for thisFile in fileList:
1156 1156
1157 1157 if not os.path.isfile(os.path.join(single_path, thisFile)):
1158 1158 continue
1159 1159
1160 1160 if not isRadarFile(thisFile):
1161 1161 continue
1162 1162
1163 1163 if not isFileInDateRange(thisFile, startDate, endDate):
1164 1164 continue
1165 1165
1166 1166 thisDate = getDateFromRadarFile(thisFile)
1167 1167
1168 1168 if thisDate in dateList:
1169 1169 continue
1170 1170
1171 1171 dateList.append(thisDate)
1172 1172 pathList.append(single_path)
1173 1173
1174 1174 else:
1175 1175 for single_path in multi_path:
1176 1176
1177 1177 if not os.path.isdir(single_path):
1178 1178 continue
1179 1179
1180 1180 dirList = []
1181 1181
1182 1182 for thisPath in os.listdir(single_path):
1183 1183
1184 1184 if not os.path.isdir(os.path.join(single_path,thisPath)):
1185 1185 continue
1186 1186
1187 1187 if not isRadarFolder(thisPath):
1188 1188 continue
1189 1189
1190 1190 if not isFolderInDateRange(thisPath, startDate, endDate):
1191 1191 continue
1192 1192
1193 1193 dirList.append(thisPath)
1194 1194
1195 1195 if not dirList:
1196 1196 continue
1197 1197
1198 1198 dirList.sort()
1199 1199
1200 1200 for thisDir in dirList:
1201 1201
1202 1202 datapath = os.path.join(single_path, thisDir, expLabel)
1203 1203 fileList = glob.glob1(datapath, "*"+ext)
1204 1204
1205 1205 if not fileList:
1206 1206 continue
1207 1207
1208 1208 path_empty = False
1209 1209
1210 1210 thisDate = getDateFromRadarFolder(thisDir)
1211 1211
1212 1212 pathList.append(datapath)
1213 1213 dateList.append(thisDate)
1214 1214
1215 1215 dateList.sort()
1216 1216
1217 1217 if walk:
1218 1218 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1219 1219 else:
1220 1220 pattern_path = multi_path[0]
1221 1221
1222 1222 if path_empty:
1223 1223 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1224 1224 else:
1225 1225 if not dateList:
1226 1226 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1227 1227
1228 1228 if include_path:
1229 1229 return dateList, pathList
1230 1230
1231 1231 return dateList
1232 1232
1233 1233 def setup(self,
1234 1234 path=None,
1235 1235 startDate=None,
1236 1236 endDate=None,
1237 1237 startTime=datetime.time(0,0,0),
1238 1238 endTime=datetime.time(23,59,59),
1239 1239 set=None,
1240 1240 expLabel = "",
1241 1241 ext = None,
1242 1242 online = False,
1243 1243 delay = 60,
1244 1244 walk = True,
1245 1245 getblock = False,
1246 1246 nTxs = 1,
1247 1247 realtime=False,
1248 1248 blocksize=None,
1249 1249 blocktime=None):
1250 1250
1251 1251 if path == None:
1252 1252 raise ValueError, "[Reading] The path is not valid"
1253 1253
1254 1254 if ext == None:
1255 1255 ext = self.ext
1256 1256
1257 1257 if online:
1258 1258 print "[Reading] Searching files in online mode..."
1259 1259
1260 1260 for nTries in range( self.nTries ):
1261 1261 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1262 1262
1263 1263 if fullpath:
1264 1264 break
1265 1265
1266 1266 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1267 1267 sleep( self.delay )
1268 1268
1269 1269 if not(fullpath):
1270 1270 print "[Reading] There 'isn't any valid file in %s" % path
1271 1271 return
1272 1272
1273 1273 self.year = year
1274 1274 self.doy = doy
1275 1275 self.set = set - 1
1276 1276 self.path = path
1277 1277 self.foldercounter = foldercounter
1278 1278 last_set = None
1279 1279
1280 1280 else:
1281 1281 print "[Reading] Searching files in offline mode ..."
1282 1282 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1283 1283 startTime=startTime, endTime=endTime,
1284 1284 set=set, expLabel=expLabel, ext=ext,
1285 1285 walk=walk)
1286 1286
1287 1287 if not(pathList):
1288 1288 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1289 1289 # datetime.datetime.combine(startDate,startTime).ctime(),
1290 1290 # datetime.datetime.combine(endDate,endTime).ctime())
1291 1291
1292 1292 # sys.exit(-1)
1293 1293
1294 1294 self.fileIndex = -1
1295 1295 self.pathList = []
1296 1296 self.filenameList = []
1297 1297 return
1298 1298
1299 1299 self.fileIndex = -1
1300 1300 self.pathList = pathList
1301 1301 self.filenameList = filenameList
1302 1302 file_name = os.path.basename(filenameList[-1])
1303 1303 basename, ext = os.path.splitext(file_name)
1304 1304 last_set = int(basename[-3:])
1305 1305
1306 1306 self.online = online
1307 1307 self.realtime = realtime
1308 1308 self.delay = delay
1309 1309 ext = ext.lower()
1310 1310 self.ext = ext
1311 1311 self.getByBlock = getblock
1312 1312 self.nTxs = nTxs
1313 1313 self.startTime = startTime
1314 1314 self.endTime = endTime
1315 1315
1316 1316 #Added-----------------
1317 1317 self.selBlocksize = blocksize
1318 1318 self.selBlocktime = blocktime
1319 1319
1320 1320
1321 1321 if not(self.setNextFile()):
1322 1322 if (startDate!=None) and (endDate!=None):
1323 1323 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1324 1324 elif startDate != None:
1325 1325 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1326 1326 else:
1327 1327 print "[Reading] No files"
1328 1328
1329 1329 self.fileIndex = -1
1330 1330 self.pathList = []
1331 1331 self.filenameList = []
1332 1332 return
1333 1333
1334 # self.getBasicHeader()
1334 # self.getBasicHeader()
1335 1335
1336 1336 if last_set != None:
1337 1337 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1338 1338 return
1339 1339
1340 1340 def getBasicHeader(self):
1341 1341
1342 1342 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1343 1343
1344 1344 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1345 1345
1346 1346 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1347 1347
1348 1348 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1349 1349
1350 1350 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1351 1351
1352 1352 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1353 1353
1354 1354 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1355 1355
1356 1356 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1357 1357
1358 1358
1359 1359 def getFirstHeader(self):
1360 1360
1361 1361 raise NotImplementedError
1362 1362
1363 1363 def getData(self):
1364 1364
1365 1365 raise NotImplementedError
1366 1366
1367 1367 def hasNotDataInBuffer(self):
1368 1368
1369 1369 raise NotImplementedError
1370 1370
1371 1371 def readBlock(self):
1372 1372
1373 1373 raise NotImplementedError
1374 1374
1375 1375 def isEndProcess(self):
1376 1376
1377 1377 return self.flagNoMoreFiles
1378 1378
1379 1379 def printReadBlocks(self):
1380 1380
1381 1381 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1382 1382
1383 1383 def printTotalBlocks(self):
1384 1384
1385 1385 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1386 1386
1387 1387 def printNumberOfBlock(self):
1388 1388
1389 1389 if self.flagIsNewBlock:
1390 1390 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1391 1391 self.processingHeaderObj.dataBlocksPerFile,
1392 1392 self.dataOut.datatime.ctime())
1393 1393
1394 1394 def printInfo(self):
1395 1395
1396 1396 if self.__printInfo == False:
1397 1397 return
1398 1398
1399 1399 self.basicHeaderObj.printInfo()
1400 1400 self.systemHeaderObj.printInfo()
1401 1401 self.radarControllerHeaderObj.printInfo()
1402 1402 self.processingHeaderObj.printInfo()
1403 1403
1404 1404 self.__printInfo = False
1405 1405
1406 1406
1407 1407 def run(self, **kwargs):
1408 1408
1409 1409 if not(self.isConfig):
1410 1410
1411 1411 # self.dataOut = dataOut
1412 1412 self.setup(**kwargs)
1413 1413 self.isConfig = True
1414 1414
1415 1415 self.getData()
1416 1416
1417 1417 class JRODataWriter(JRODataIO):
1418 1418
1419 1419 """
1420 1420 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1421 1421 de los datos siempre se realiza por bloques.
1422 1422 """
1423 1423
1424 1424 blockIndex = 0
1425 1425
1426 1426 path = None
1427 1427
1428 1428 setFile = None
1429 1429
1430 1430 profilesPerBlock = None
1431 1431
1432 1432 blocksPerFile = None
1433 1433
1434 1434 nWriteBlocks = 0
1435 1435
1436 1436 fileDate = None
1437 1437
1438 1438 def __init__(self, dataOut=None):
1439 1439 raise NotImplementedError
1440 1440
1441 1441
1442 1442 def hasAllDataInBuffer(self):
1443 1443 raise NotImplementedError
1444 1444
1445 1445
1446 1446 def setBlockDimension(self):
1447 1447 raise NotImplementedError
1448 1448
1449 1449
1450 1450 def writeBlock(self):
1451 1451 raise NotImplementedError
1452 1452
1453 1453
1454 1454 def putData(self):
1455 1455 raise NotImplementedError
1456 1456
1457 1457
1458 1458 def getProcessFlags(self):
1459 1459
1460 1460 processFlags = 0
1461 1461
1462 1462 dtype_index = get_dtype_index(self.dtype)
1463 1463 procflag_dtype = get_procflag_dtype(dtype_index)
1464 1464
1465 1465 processFlags += procflag_dtype
1466 1466
1467 1467 if self.dataOut.flagDecodeData:
1468 1468 processFlags += PROCFLAG.DECODE_DATA
1469 1469
1470 1470 if self.dataOut.flagDeflipData:
1471 1471 processFlags += PROCFLAG.DEFLIP_DATA
1472 1472
1473 1473 if self.dataOut.code is not None:
1474 1474 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1475 1475
1476 1476 if self.dataOut.nCohInt > 1:
1477 1477 processFlags += PROCFLAG.COHERENT_INTEGRATION
1478 1478
1479 1479 if self.dataOut.type == "Spectra":
1480 1480 if self.dataOut.nIncohInt > 1:
1481 1481 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1482 1482
1483 1483 if self.dataOut.data_dc is not None:
1484 1484 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1485 1485
1486 1486 if self.dataOut.flagShiftFFT:
1487 1487 processFlags += PROCFLAG.SHIFT_FFT_DATA
1488 1488
1489 1489 return processFlags
1490 1490
1491 1491 def setBasicHeader(self):
1492 1492
1493 1493 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1494 1494 self.basicHeaderObj.version = self.versionFile
1495 1495 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1496 1496
1497 1497 utc = numpy.floor(self.dataOut.utctime)
1498 1498 milisecond = (self.dataOut.utctime - utc)* 1000.0
1499 1499
1500 1500 self.basicHeaderObj.utc = utc
1501 1501 self.basicHeaderObj.miliSecond = milisecond
1502 1502 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1503 1503 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1504 1504 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1505 1505
1506 1506 def setFirstHeader(self):
1507 1507 """
1508 1508 Obtiene una copia del First Header
1509 1509
1510 1510 Affected:
1511 1511
1512 1512 self.basicHeaderObj
1513 1513 self.systemHeaderObj
1514 1514 self.radarControllerHeaderObj
1515 1515 self.processingHeaderObj self.
1516 1516
1517 1517 Return:
1518 1518 None
1519 1519 """
1520 1520
1521 1521 raise NotImplementedError
1522 1522
1523 1523 def __writeFirstHeader(self):
1524 1524 """
1525 1525 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1526 1526
1527 1527 Affected:
1528 1528 __dataType
1529 1529
1530 1530 Return:
1531 1531 None
1532 1532 """
1533 1533
1534 1534 # CALCULAR PARAMETROS
1535 1535
1536 1536 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1537 1537 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1538 1538
1539 1539 self.basicHeaderObj.write(self.fp)
1540 1540 self.systemHeaderObj.write(self.fp)
1541 1541 self.radarControllerHeaderObj.write(self.fp)
1542 1542 self.processingHeaderObj.write(self.fp)
1543 1543
1544 1544 def __setNewBlock(self):
1545 1545 """
1546 1546 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1547 1547
1548 1548 Return:
1549 1549 0 : si no pudo escribir nada
1550 1550 1 : Si escribio el Basic el First Header
1551 1551 """
1552 1552 if self.fp == None:
1553 1553 self.setNextFile()
1554 1554
1555 1555 if self.flagIsNewFile:
1556 1556 return 1
1557 1557
1558 1558 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1559 1559 self.basicHeaderObj.write(self.fp)
1560 1560 return 1
1561 1561
1562 1562 if not( self.setNextFile() ):
1563 1563 return 0
1564 1564
1565 1565 return 1
1566 1566
1567 1567
1568 1568 def writeNextBlock(self):
1569 1569 """
1570 1570 Selecciona el bloque siguiente de datos y los escribe en un file
1571 1571
1572 1572 Return:
1573 1573 0 : Si no hizo pudo escribir el bloque de datos
1574 1574 1 : Si no pudo escribir el bloque de datos
1575 1575 """
1576 1576 if not( self.__setNewBlock() ):
1577 1577 return 0
1578 1578
1579 1579 self.writeBlock()
1580 1580
1581 1581 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1582 1582 self.processingHeaderObj.dataBlocksPerFile)
1583 1583
1584 1584 return 1
1585 1585
1586 1586 def setNextFile(self):
1587 1587 """
1588 1588 Determina el siguiente file que sera escrito
1589 1589
1590 1590 Affected:
1591 1591 self.filename
1592 1592 self.subfolder
1593 1593 self.fp
1594 1594 self.setFile
1595 1595 self.flagIsNewFile
1596 1596
1597 1597 Return:
1598 1598 0 : Si el archivo no puede ser escrito
1599 1599 1 : Si el archivo esta listo para ser escrito
1600 1600 """
1601 1601 ext = self.ext
1602 1602 path = self.path
1603 1603
1604 1604 if self.fp != None:
1605 1605 self.fp.close()
1606 1606
1607 1607 timeTuple = time.localtime( self.dataOut.utctime)
1608 1608 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1609 1609
1610 1610 fullpath = os.path.join( path, subfolder )
1611 1611 setFile = self.setFile
1612 1612
1613 1613 if not( os.path.exists(fullpath) ):
1614 1614 os.mkdir(fullpath)
1615 1615 setFile = -1 #inicializo mi contador de seteo
1616 1616 else:
1617 1617 filesList = os.listdir( fullpath )
1618 1618 if len( filesList ) > 0:
1619 1619 filesList = sorted( filesList, key=str.lower )
1620 1620 filen = filesList[-1]
1621 1621 # el filename debera tener el siguiente formato
1622 1622 # 0 1234 567 89A BCDE (hex)
1623 1623 # x YYYY DDD SSS .ext
1624 1624 if isNumber( filen[8:11] ):
1625 1625 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1626 1626 else:
1627 1627 setFile = -1
1628 1628 else:
1629 1629 setFile = -1 #inicializo mi contador de seteo
1630 1630
1631 1631 setFile += 1
1632 1632
1633 1633 #If this is a new day it resets some values
1634 1634 if self.dataOut.datatime.date() > self.fileDate:
1635 1635 setFile = 0
1636 1636 self.nTotalBlocks = 0
1637 1637
1638 1638 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1639 1639
1640 1640 filename = os.path.join( path, subfolder, filen )
1641 1641
1642 1642 fp = open( filename,'wb' )
1643 1643
1644 1644 self.blockIndex = 0
1645 1645
1646 1646 #guardando atributos
1647 1647 self.filename = filename
1648 1648 self.subfolder = subfolder
1649 1649 self.fp = fp
1650 1650 self.setFile = setFile
1651 1651 self.flagIsNewFile = 1
1652 1652 self.fileDate = self.dataOut.datatime.date()
1653 1653
1654 1654 self.setFirstHeader()
1655 1655
1656 1656 print '[Writing] Opening file: %s'%self.filename
1657 1657
1658 1658 self.__writeFirstHeader()
1659 1659
1660 1660 return 1
1661 1661
1662 1662 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1663 1663 """
1664 1664 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1665 1665
1666 1666 Inputs:
1667 1667 path : directory where data will be saved
1668 1668 profilesPerBlock : number of profiles per block
1669 1669 set : initial file set
1670 1670 datatype : An integer number that defines data type:
1671 1671 0 : int8 (1 byte)
1672 1672 1 : int16 (2 bytes)
1673 1673 2 : int32 (4 bytes)
1674 1674 3 : int64 (8 bytes)
1675 1675 4 : float32 (4 bytes)
1676 1676 5 : double64 (8 bytes)
1677 1677
1678 1678 Return:
1679 1679 0 : Si no realizo un buen seteo
1680 1680 1 : Si realizo un buen seteo
1681 1681 """
1682 1682
1683 1683 if ext == None:
1684 1684 ext = self.ext
1685 1685
1686 1686 self.ext = ext.lower()
1687 1687
1688 1688 self.path = path
1689 1689
1690 1690 if set is None:
1691 1691 self.setFile = -1
1692 1692 else:
1693 1693 self.setFile = set - 1
1694 1694
1695 1695 self.blocksPerFile = blocksPerFile
1696 1696
1697 1697 self.profilesPerBlock = profilesPerBlock
1698 1698
1699 1699 self.dataOut = dataOut
1700 1700 self.fileDate = self.dataOut.datatime.date()
1701 1701 #By default
1702 1702 self.dtype = self.dataOut.dtype
1703 1703
1704 1704 if datatype is not None:
1705 1705 self.dtype = get_numpy_dtype(datatype)
1706 1706
1707 1707 if not(self.setNextFile()):
1708 1708 print "[Writing] There isn't a next file"
1709 1709 return 0
1710 1710
1711 1711 self.setBlockDimension()
1712 1712
1713 1713 return 1
1714 1714
1715 1715 def run(self, dataOut, **kwargs):
1716 1716
1717 1717 if not(self.isConfig):
1718 1718
1719 1719 self.setup(dataOut, **kwargs)
1720 1720 self.isConfig = True
1721 1721
1722 1722 self.putData()
1723 1723
General Comments 0
You need to be logged in to leave comments. Login now