##// END OF EJS Templates
schain-jc
RSS
arreglado varios dias con mp
José Chávez -
r1079:77593f0c5ecb
parent child
Show More
Side by Side Unified
Context file:
r1079:77593f0c5ecb Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,86 +1,88
1 1 basic = '''from schainpy.controller import Project
2 2
3 3 desc = "{desc}"
4 4 project = Project()
5 5 project.setup(id='200', name="{name}", description=desc)
6 6
7 7 voltage_reader = project.addReadUnit(datatype='VoltageReader',
8 8 path="{path}",
9 9 startDate="{startDate}",
10 10 endDate="{endDate}",
11 11 startTime="{startHour}",
12 12 endTime="{endHour}",
13 13 online=0,
14 14 verbose=1,
15 15 walk=1,
16 16 )
17 17
18 18 voltage_proc = project.addProcUnit(datatype='VoltageProc', inputId=voltage_reader.getId())
19 19
20 20 profile = voltage_proc.addOperation(name='ProfileSelector', optype='other')
21 21 profile.addParameter(name='profileRangeList', value='120,183', format='intlist')
22 22
23 23 rti = voltage_proc.addOperation(name='RTIPlot', optype='other')
24 24 rti.addParameter(name='wintitle', value='Jicamarca Radio Observatory', format='str')
25 25 rti.addParameter(name='showprofile', value='0', format='int')
26 26 rti.addParameter(name='xmin', value='0', format='int')
27 27 rti.addParameter(name='xmax', value='24', format='int')
28 28 rti.addParameter(name='figpath', value="{figpath}", format='str')
29 29 rti.addParameter(name='wr_period', value='5', format='int')
30 30 rti.addParameter(name='exp_code', value='22', format='int')
31 31
32 32
33 33 controller.start()
34 34 '''
35 35
36 36 multiprocess = '''from schainpy.controller import Project, MPProject
37
37 from time import sleep
38 38 desc = "{desc}"
39 39
40 40 ####################
41 41 # PLOTTER RECEIVER #
42 42 ####################
43 43 plotter = Project()
44 44 plotter.setup(id='100', name='receiver', description=desc)
45 45
46 46 receiver_proc = plotter.addProcUnit(name='PlotterReceiver')
47 47 receiver_proc.addParameter(name='throttle', value=20, format='int')
48 48
49 49 rti = receiver_proc.addOperation(name='PlotRTIData', optype='other')
50 50 rti.addParameter(name='zmin', value='-40.0', format='float')
51 51 rti.addParameter(name='zmax', value='100.0', format='float')
52 52 rti.addParameter(name='xmin', value='0.0', format='int')
53 53 rti.addParameter(name='colormap', value='jet', format='str')
54 54
55 55 plotter.start()
56 56
57 sleep(2)
58
57 59 ################
58 60 # DATA EMITTER #
59 61 ################
60 62 project = Project()
61 63 project.setup(id='200', name="{name}", description=desc)
62 64
63 65 spectra_reader = project.addReadUnit(datatype='SpectraReader',
64 66 path="{path}",
65 67 startDate={startDate},
66 68 endDate={endDate},
67 69 startTime="{startHour}",
68 70 endTime="{endHour}",
69 71 online=0,
70 72 verbose=1,
71 73 walk=1,
72 74 )
73 75
74 76 spectra_proc = project.addProcUnit(datatype='Spectra', inputId=spectra_reader.getId())
75 77
76 78 parameters_proc = project.addProcUnit(datatype='ParametersProc', inputId=spectra_proc.getId())
77 79 moments = parameters_proc.addOperation(name='SpectralMoments', optype='other')
78 80
79 81 publish = parameters_proc.addOperation(name='PublishData', optype='other')
80 82 publish.addParameter(name='zeromq', value=1, format='int')
81 83 publish.addParameter(name='verbose', value=0, format='bool')
82 84
83 85 MPProject(project, 16)
84 86
85 87
86 88 '''
Show file before | Show file after | Hide comments
@@ -1,1834 +1,1830
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
24 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
25 25
26 26 LOCALTIME = True
27 27
28 28
29 29 def isNumber(cad):
30 30 """
31 31 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
32 32
33 33 Excepciones:
34 34 Si un determinado string no puede ser convertido a numero
35 35 Input:
36 36 str, string al cual se le analiza para determinar si convertible a un numero o no
37 37
38 38 Return:
39 39 True : si el string es uno numerico
40 40 False : no es un string numerico
41 41 """
42 42 try:
43 43 float(cad)
44 44 return True
45 45 except:
46 46 return False
47 47
48 48
49 49 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
50 50 """
51 51 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
52 52
53 53 Inputs:
54 54 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
55 55
56 56 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
57 57 segundos contados desde 01/01/1970.
58 58 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
59 59 segundos contados desde 01/01/1970.
60 60
61 61 Return:
62 62 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
63 63 fecha especificado, de lo contrario retorna False.
64 64
65 65 Excepciones:
66 66 Si el archivo no existe o no puede ser abierto
67 67 Si la cabecera no puede ser leida.
68 68
69 69 """
70 70 basicHeaderObj = BasicHeader(LOCALTIME)
71 71
72 72 try:
73 73 fp = open(filename, 'rb')
74 74 except IOError:
75 75 print "The file %s can't be opened" % (filename)
76 76 return 0
77 77
78 78 sts = basicHeaderObj.read(fp)
79 79 fp.close()
80 80
81 81 if not(sts):
82 82 print "Skipping the file %s because it has not a valid header" % (filename)
83 83 return 0
84 84
85 85 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
86 86 return 0
87 87
88 88 return 1
89 89
90 90
91 91 def isTimeInRange(thisTime, startTime, endTime):
92
93 92 if endTime >= startTime:
94 93 if (thisTime < startTime) or (thisTime > endTime):
95 94 return 0
96
97 95 return 1
98 96 else:
99 97 if (thisTime < startTime) and (thisTime > endTime):
100 98 return 0
101
102 99 return 1
103 100
104 101
105 102 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
106 103 """
107 104 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
108 105
109 106 Inputs:
110 107 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
111 108
112 109 startDate : fecha inicial del rango seleccionado en formato datetime.date
113 110
114 111 endDate : fecha final del rango seleccionado en formato datetime.date
115 112
116 113 startTime : tiempo inicial del rango seleccionado en formato datetime.time
117 114
118 115 endTime : tiempo final del rango seleccionado en formato datetime.time
119 116
120 117 Return:
121 118 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
122 119 fecha especificado, de lo contrario retorna False.
123 120
124 121 Excepciones:
125 122 Si el archivo no existe o no puede ser abierto
126 123 Si la cabecera no puede ser leida.
127 124
128 125 """
129 126
130 127 try:
131 128 fp = open(filename, 'rb')
132 129 except IOError:
133 130 print "The file %s can't be opened" % (filename)
134 131 return None
135 132
136 133 firstBasicHeaderObj = BasicHeader(LOCALTIME)
137 134 systemHeaderObj = SystemHeader()
138 135 radarControllerHeaderObj = RadarControllerHeader()
139 136 processingHeaderObj = ProcessingHeader()
140 137
141 138 lastBasicHeaderObj = BasicHeader(LOCALTIME)
142 139
143 140 sts = firstBasicHeaderObj.read(fp)
144 141
145 142 if not(sts):
146 143 print "[Reading] Skipping the file %s because it has not a valid header" % (filename)
147 144 return None
148 145
149 146 if not systemHeaderObj.read(fp):
150 147 return None
151 148
152 149 if not radarControllerHeaderObj.read(fp):
153 150 return None
154 151
155 152 if not processingHeaderObj.read(fp):
156 153 return None
157 154
158 155 filesize = os.path.getsize(filename)
159 156
160 157 offset = processingHeaderObj.blockSize + 24 # header size
161 158
162 159 if filesize <= offset:
163 160 print "[Reading] %s: This file has not enough data" % filename
164 161 return None
165 162
166 163 fp.seek(-offset, 2)
167 164
168 165 sts = lastBasicHeaderObj.read(fp)
169 166
170 167 fp.close()
171 168
172 169 thisDatetime = lastBasicHeaderObj.datatime
173 170 thisTime_last_block = thisDatetime.time()
174 171
175 172 thisDatetime = firstBasicHeaderObj.datatime
176 173 thisDate = thisDatetime.date()
177 174 thisTime_first_block = thisDatetime.time()
178 175
179 176 # General case
180 177 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
181 178 #-----------o----------------------------o-----------
182 179 # startTime endTime
183 180
184 181 if endTime >= startTime:
185 182 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
186 183 return None
187 184
188 185 return thisDatetime
189 186
190 187 # If endTime < startTime then endTime belongs to the next day
191 188
192 189 #<<<<<<<<<<<o o>>>>>>>>>>>
193 190 #-----------o----------------------------o-----------
194 191 # endTime startTime
195 192
196 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
197 194 return None
198 195
199 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
200 197 return None
201 198
202 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
203 200 return None
204 201
205 202 return thisDatetime
206 203
207 204
208 205 def isFolderInDateRange(folder, startDate=None, endDate=None):
209 206 """
210 207 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
211 208
212 209 Inputs:
213 210 folder : nombre completo del directorio.
214 211 Su formato deberia ser "/path_root/?YYYYDDD"
215 212
216 213 siendo:
217 214 YYYY : Anio (ejemplo 2015)
218 215 DDD : Dia del anio (ejemplo 305)
219 216
220 217 startDate : fecha inicial del rango seleccionado en formato datetime.date
221 218
222 219 endDate : fecha final del rango seleccionado en formato datetime.date
223 220
224 221 Return:
225 222 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
226 223 fecha especificado, de lo contrario retorna False.
227 224 Excepciones:
228 225 Si el directorio no tiene el formato adecuado
229 226 """
230 227
231 228 basename = os.path.basename(folder)
232 229
233 230 if not isRadarFolder(basename):
234 231 print "The folder %s has not the rigth format" % folder
235 232 return 0
236 233
237 234 if startDate and endDate:
238 235 thisDate = getDateFromRadarFolder(basename)
239 236
240 237 if thisDate < startDate:
241 238 return 0
242 239
243 240 if thisDate > endDate:
244 241 return 0
245 242
246 243 return 1
247 244
248 245
249 246 def isFileInDateRange(filename, startDate=None, endDate=None):
250 247 """
251 248 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
252 249
253 250 Inputs:
254 251 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
255 252
256 253 Su formato deberia ser "?YYYYDDDsss"
257 254
258 255 siendo:
259 256 YYYY : Anio (ejemplo 2015)
260 257 DDD : Dia del anio (ejemplo 305)
261 258 sss : set
262 259
263 260 startDate : fecha inicial del rango seleccionado en formato datetime.date
264 261
265 262 endDate : fecha final del rango seleccionado en formato datetime.date
266 263
267 264 Return:
268 265 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
269 266 fecha especificado, de lo contrario retorna False.
270 267 Excepciones:
271 268 Si el archivo no tiene el formato adecuado
272 269 """
273 270
274 271 basename = os.path.basename(filename)
275 272
276 273 if not isRadarFile(basename):
277 274 print "The filename %s has not the rigth format" % filename
278 275 return 0
279 276
280 277 if startDate and endDate:
281 278 thisDate = getDateFromRadarFile(basename)
282 279
283 280 if thisDate < startDate:
284 281 return 0
285 282
286 283 if thisDate > endDate:
287 284 return 0
288 285
289 286 return 1
290 287
291 288
292 289 def getFileFromSet(path, ext, set):
293 290 validFilelist = []
294 291 fileList = os.listdir(path)
295 292
296 293 # 0 1234 567 89A BCDE
297 294 # H YYYY DDD SSS .ext
298 295
299 296 for thisFile in fileList:
300 297 try:
301 298 year = int(thisFile[1:5])
302 299 doy = int(thisFile[5:8])
303 300 except:
304 301 continue
305 302
306 303 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 304 continue
308 305
309 306 validFilelist.append(thisFile)
310 307
311 308 myfile = fnmatch.filter(
312 309 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
313 310
314 311 if len(myfile) != 0:
315 312 return myfile[0]
316 313 else:
317 314 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
318 315 print 'the filename %s does not exist' % filename
319 316 print '...going to the last file: '
320 317
321 318 if validFilelist:
322 319 validFilelist = sorted(validFilelist, key=str.lower)
323 320 return validFilelist[-1]
324 321
325 322 return None
326 323
327 324
328 325 def getlastFileFromPath(path, ext):
329 326 """
330 327 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
331 328 al final de la depuracion devuelve el ultimo file de la lista que quedo.
332 329
333 330 Input:
334 331 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
335 332 ext : extension de los files contenidos en una carpeta
336 333
337 334 Return:
338 335 El ultimo file de una determinada carpeta, no se considera el path.
339 336 """
340 337 validFilelist = []
341 338 fileList = os.listdir(path)
342 339
343 340 # 0 1234 567 89A BCDE
344 341 # H YYYY DDD SSS .ext
345 342
346 343 for thisFile in fileList:
347 344
348 345 year = thisFile[1:5]
349 346 if not isNumber(year):
350 347 continue
351 348
352 349 doy = thisFile[5:8]
353 350 if not isNumber(doy):
354 351 continue
355 352
356 353 year = int(year)
357 354 doy = int(doy)
358 355
359 356 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
360 357 continue
361 358
362 359 validFilelist.append(thisFile)
363 360
364 361 if validFilelist:
365 362 validFilelist = sorted(validFilelist, key=str.lower)
366 363 return validFilelist[-1]
367 364
368 365 return None
369 366
370 367
371 368 def checkForRealPath(path, foldercounter, year, doy, set, ext):
372 369 """
373 370 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
374 371 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
375 372 el path exacto de un determinado file.
376 373
377 374 Example :
378 375 nombre correcto del file es .../.../D2009307/P2009307367.ext
379 376
380 377 Entonces la funcion prueba con las siguientes combinaciones
381 378 .../.../y2009307367.ext
382 379 .../.../Y2009307367.ext
383 380 .../.../x2009307/y2009307367.ext
384 381 .../.../x2009307/Y2009307367.ext
385 382 .../.../X2009307/y2009307367.ext
386 383 .../.../X2009307/Y2009307367.ext
387 384 siendo para este caso, la ultima combinacion de letras, identica al file buscado
388 385
389 386 Return:
390 387 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
391 388 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
392 389 para el filename
393 390 """
394 391 fullfilename = None
395 392 find_flag = False
396 393 filename = None
397 394
398 395 prefixDirList = [None, 'd', 'D']
399 396 if ext.lower() == ".r": # voltage
400 397 prefixFileList = ['d', 'D']
401 398 elif ext.lower() == ".pdata": # spectra
402 399 prefixFileList = ['p', 'P']
403 400 else:
404 401 return None, filename
405 402
406 403 # barrido por las combinaciones posibles
407 404 for prefixDir in prefixDirList:
408 405 thispath = path
409 406 if prefixDir != None:
410 407 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
411 408 if foldercounter == 0:
412 409 thispath = os.path.join(path, "%s%04d%03d" %
413 410 (prefixDir, year, doy))
414 411 else:
415 412 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
416 413 prefixDir, year, doy, foldercounter))
417 414 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
418 415 # formo el nombre del file xYYYYDDDSSS.ext
419 416 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
420 417 fullfilename = os.path.join(
421 418 thispath, filename) # formo el path completo
422 419
423 420 if os.path.exists(fullfilename): # verifico que exista
424 421 find_flag = True
425 422 break
426 423 if find_flag:
427 424 break
428 425
429 426 if not(find_flag):
430 427 return None, filename
431 428
432 429 return fullfilename, filename
433 430
434 431
435 432 def isRadarFolder(folder):
436 433 try:
437 434 year = int(folder[1:5])
438 435 doy = int(folder[5:8])
439 436 except:
440 437 return 0
441 438
442 439 return 1
443 440
444 441
445 442 def isRadarFile(file):
446 443 try:
447 444 year = int(file[1:5])
448 445 doy = int(file[5:8])
449 446 set = int(file[8:11])
450 447 except:
451 448 return 0
452 449
453 450 return 1
454 451
455 452
456 453 def getDateFromRadarFile(file):
457 454 try:
458 455 year = int(file[1:5])
459 456 doy = int(file[5:8])
460 457 set = int(file[8:11])
461 458 except:
462 459 return None
463 460
464 461 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
465 462 return thisDate
466 463
467 464
468 465 def getDateFromRadarFolder(folder):
469 466 try:
470 467 year = int(folder[1:5])
471 468 doy = int(folder[5:8])
472 469 except:
473 470 return None
474 471
475 472 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
476 473 return thisDate
477 474
478 475
479 476 class JRODataIO:
480 477
481 478 c = 3E8
482 479
483 480 isConfig = False
484 481
485 482 basicHeaderObj = None
486 483
487 484 systemHeaderObj = None
488 485
489 486 radarControllerHeaderObj = None
490 487
491 488 processingHeaderObj = None
492 489
493 490 dtype = None
494 491
495 492 pathList = []
496 493
497 494 filenameList = []
498 495
499 496 filename = None
500 497
501 498 ext = None
502 499
503 500 flagIsNewFile = 1
504 501
505 502 flagDiscontinuousBlock = 0
506 503
507 504 flagIsNewBlock = 0
508 505
509 506 fp = None
510 507
511 508 firstHeaderSize = 0
512 509
513 510 basicHeaderSize = 24
514 511
515 512 versionFile = 1103
516 513
517 514 fileSize = None
518 515
519 516 # ippSeconds = None
520 517
521 518 fileSizeByHeader = None
522 519
523 520 fileIndex = None
524 521
525 522 profileIndex = None
526 523
527 524 blockIndex = None
528 525
529 526 nTotalBlocks = None
530 527
531 528 maxTimeStep = 30
532 529
533 530 lastUTTime = None
534 531
535 532 datablock = None
536 533
537 534 dataOut = None
538 535
539 536 blocksize = None
540 537
541 538 getByBlock = False
542 539
543 540 def __init__(self):
544 541
545 542 raise NotImplementedError
546 543
547 544 def run(self):
548 545
549 546 raise NotImplementedError
550 547
551 548 def getDtypeWidth(self):
552 549
553 550 dtype_index = get_dtype_index(self.dtype)
554 551 dtype_width = get_dtype_width(dtype_index)
555 552
556 553 return dtype_width
557 554
558 555 def getAllowedArgs(self):
559 556 return inspect.getargspec(self.run).args
560 557
561 558
562 559 class JRODataReader(JRODataIO):
563 560
564 561 online = 0
565 562
566 563 realtime = 0
567 564
568 565 nReadBlocks = 0
569 566
570 567 delay = 10 # number of seconds waiting a new file
571 568
572 569 nTries = 3 # quantity tries
573 570
574 571 nFiles = 3 # number of files for searching
575 572
576 573 path = None
577 574
578 575 foldercounter = 0
579 576
580 577 flagNoMoreFiles = 0
581 578
582 579 datetimeList = []
583 580
584 581 __isFirstTimeOnline = 1
585 582
586 583 __printInfo = True
587 584
588 585 profileIndex = None
589 586
590 587 nTxs = 1
591 588
592 589 txIndex = None
593 590
594 591 # Added--------------------
595 592
596 593 selBlocksize = None
597 594
598 595 selBlocktime = None
599 596
600 597 def __init__(self):
601 598 """
602 599 This class is used to find data files
603 600
604 601 Example:
605 602 reader = JRODataReader()
606 603 fileList = reader.findDataFiles()
607 604
608 605 """
609 606 pass
610 607
611 608 def createObjByDefault(self):
612 609 """
613 610
614 611 """
615 612 raise NotImplementedError
616 613
617 614 def getBlockDimension(self):
618 615
619 616 raise NotImplementedError
620 617
621 618 def searchFilesOffLine(self,
622 619 path,
623 620 startDate=None,
624 621 endDate=None,
625 622 startTime=datetime.time(0, 0, 0),
626 623 endTime=datetime.time(23, 59, 59),
627 624 set=None,
628 625 expLabel='',
629 626 ext='.r',
630 627 cursor=None,
631 628 skip=None,
632 629 walk=True):
633 630
634 631 self.filenameList = []
635 632 self.datetimeList = []
636 633
637 634 pathList = []
638 635
639 636 dateList, pathList = self.findDatafiles(
640 637 path, startDate, endDate, expLabel, ext, walk, include_path=True)
641 638
642 639 if dateList == []:
643 640 return [], []
644 641
645 642 if len(dateList) > 1:
646 643 print "[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList))
647 644 else:
648 645 print "[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0])
649 646
650 647 filenameList = []
651 648 datetimeList = []
652 649
653 650 for thisPath in pathList:
654 651
655 652 fileList = glob.glob1(thisPath, "*%s" % ext)
656 653 fileList.sort()
657 654
658 655 skippedFileList = []
659 656
660 657 if cursor is not None and skip is not None:
661 658
662 659 if skip == 0:
663 660 skippedFileList = []
664 661 else:
665 662 skippedFileList = fileList[cursor *
666 663 skip: cursor * skip + skip]
667 664
668 665 else:
669 666 skippedFileList = fileList
670 667
671 668 for file in skippedFileList:
672 669
673 670 filename = os.path.join(thisPath, file)
674 671
675 672 if not isFileInDateRange(filename, startDate, endDate):
676 673 continue
677 674
678 675 thisDatetime = isFileInTimeRange(
679 676 filename, startDate, endDate, startTime, endTime)
680 677
681 678 if not(thisDatetime):
682 679 continue
683 680
684 681 filenameList.append(filename)
685 682 datetimeList.append(thisDatetime)
686 683
687 684 if not(filenameList):
688 685 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
689 686 return [], []
690 687
691 688 print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
692 689 print
693 690
694 691 # for i in range(len(filenameList)):
695 692 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
696 693
697 694 self.filenameList = filenameList
698 695 self.datetimeList = datetimeList
699 696
700 697 return pathList, filenameList
701 698
702 699 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
703 700 """
704 701 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
705 702 devuelve el archivo encontrado ademas de otros datos.
706 703
707 704 Input:
708 705 path : carpeta donde estan contenidos los files que contiene data
709 706
710 707 expLabel : Nombre del subexperimento (subfolder)
711 708
712 709 ext : extension de los files
713 710
714 711 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
715 712
716 713 Return:
717 714 directory : eL directorio donde esta el file encontrado
718 715 filename : el ultimo file de una determinada carpeta
719 716 year : el anho
720 717 doy : el numero de dia del anho
721 718 set : el set del archivo
722 719
723 720
724 721 """
725 722 if not os.path.isdir(path):
726 723 return None, None, None, None, None, None
727 724
728 725 dirList = []
729 726
730 727 if not walk:
731 728 fullpath = path
732 729 foldercounter = 0
733 730 else:
734 731 # Filtra solo los directorios
735 732 for thisPath in os.listdir(path):
736 733 if not os.path.isdir(os.path.join(path, thisPath)):
737 734 continue
738 735 if not isRadarFolder(thisPath):
739 736 continue
740 737
741 738 dirList.append(thisPath)
742 739
743 740 if not(dirList):
744 741 return None, None, None, None, None, None
745 742
746 743 dirList = sorted(dirList, key=str.lower)
747 744
748 745 doypath = dirList[-1]
749 746 foldercounter = int(doypath.split('_')[1]) if len(
750 747 doypath.split('_')) > 1 else 0
751 748 fullpath = os.path.join(path, doypath, expLabel)
752 749
753 750 print "[Reading] %s folder was found: " % (fullpath)
754 751
755 752 if set == None:
756 753 filename = getlastFileFromPath(fullpath, ext)
757 754 else:
758 755 filename = getFileFromSet(fullpath, ext, set)
759 756
760 757 if not(filename):
761 758 return None, None, None, None, None, None
762 759
763 760 print "[Reading] %s file was found" % (filename)
764 761
765 762 if not(self.__verifyFile(os.path.join(fullpath, filename))):
766 763 return None, None, None, None, None, None
767 764
768 765 year = int(filename[1:5])
769 766 doy = int(filename[5:8])
770 767 set = int(filename[8:11])
771 768
772 769 return fullpath, foldercounter, filename, year, doy, set
773 770
774 771 def __setNextFileOffline(self):
775 772
776 773 idFile = self.fileIndex
777 774
778 775 while (True):
779 776 idFile += 1
780 777 if not(idFile < len(self.filenameList)):
781 778 self.flagNoMoreFiles = 1
782 779 # print "[Reading] No more Files"
783 780 return 0
784 781
785 782 filename = self.filenameList[idFile]
786 783
787 784 if not(self.__verifyFile(filename)):
788 785 continue
789 786
790 787 fileSize = os.path.getsize(filename)
791 788 fp = open(filename, 'rb')
792 789 break
793 790
794 791 self.flagIsNewFile = 1
795 792 self.fileIndex = idFile
796 793 self.filename = filename
797 794 self.fileSize = fileSize
798 795 self.fp = fp
799 796
800 797 # print "[Reading] Setting the file: %s"%self.filename
801 798
802 799 return 1
803 800
804 801 def __setNextFileOnline(self):
805 802 """
806 803 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
807 804 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
808 805 siguientes.
809 806
810 807 Affected:
811 808 self.flagIsNewFile
812 809 self.filename
813 810 self.fileSize
814 811 self.fp
815 812 self.set
816 813 self.flagNoMoreFiles
817 814
818 815 Return:
819 816 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
820 817 1 : si el file fue abierto con exito y esta listo a ser leido
821 818
822 819 Excepciones:
823 820 Si un determinado file no puede ser abierto
824 821 """
825 822 nFiles = 0
826 823 fileOk_flag = False
827 824 firstTime_flag = True
828 825
829 826 self.set += 1
830 827
831 828 if self.set > 999:
832 829 self.set = 0
833 830 self.foldercounter += 1
834 831
835 832 # busca el 1er file disponible
836 833 fullfilename, filename = checkForRealPath(
837 834 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
838 835 if fullfilename:
839 836 if self.__verifyFile(fullfilename, False):
840 837 fileOk_flag = True
841 838
842 839 # si no encuentra un file entonces espera y vuelve a buscar
843 840 if not(fileOk_flag):
844 841 # busco en los siguientes self.nFiles+1 files posibles
845 842 for nFiles in range(self.nFiles + 1):
846 843
847 844 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
848 845 tries = self.nTries
849 846 else:
850 847 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
851 848
852 849 for nTries in range(tries):
853 850 if firstTime_flag:
854 851 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1)
855 852 sleep(self.delay)
856 853 else:
857 854 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
858 855
859 856 fullfilename, filename = checkForRealPath(
860 857 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
861 858 if fullfilename:
862 859 if self.__verifyFile(fullfilename):
863 860 fileOk_flag = True
864 861 break
865 862
866 863 if fileOk_flag:
867 864 break
868 865
869 866 firstTime_flag = False
870 867
871 868 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
872 869 self.set += 1
873 870
874 871 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
875 872 if nFiles == (self.nFiles - 1):
876 873 self.set = 0
877 874 self.doy += 1
878 875 self.foldercounter = 0
879 876
880 877 if fileOk_flag:
881 878 self.fileSize = os.path.getsize(fullfilename)
882 879 self.filename = fullfilename
883 880 self.flagIsNewFile = 1
884 881 if self.fp != None:
885 882 self.fp.close()
886 883 self.fp = open(fullfilename, 'rb')
887 884 self.flagNoMoreFiles = 0
888 885 # print '[Reading] Setting the file: %s' % fullfilename
889 886 else:
890 887 self.fileSize = 0
891 888 self.filename = None
892 889 self.flagIsNewFile = 0
893 890 self.fp = None
894 891 self.flagNoMoreFiles = 1
895 892 # print '[Reading] No more files to read'
896 893
897 894 return fileOk_flag
898 895
899 896 def setNextFile(self):
900 897 if self.fp != None:
901 898 self.fp.close()
902 899
903 900 if self.online:
904 901 newFile = self.__setNextFileOnline()
905 902 else:
906 903 newFile = self.__setNextFileOffline()
907 904
908 905 if not(newFile):
909 906 print '[Reading] No more files to read'
910 907 return 0
911 908
912 909 if self.verbose:
913 910 print '[Reading] Setting the file: %s' % self.filename
914 911
915 912 self.__readFirstHeader()
916 913 self.nReadBlocks = 0
917 914 return 1
918 915
919 916 def __waitNewBlock(self):
920 917 """
921 918 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
922 919
923 920 Si el modo de lectura es OffLine siempre retorn 0
924 921 """
925 922 if not self.online:
926 923 return 0
927 924
928 925 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
929 926 return 0
930 927
931 928 currentPointer = self.fp.tell()
932 929
933 930 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
934 931
935 932 for nTries in range(self.nTries):
936 933
937 934 self.fp.close()
938 935 self.fp = open(self.filename, 'rb')
939 936 self.fp.seek(currentPointer)
940 937
941 938 self.fileSize = os.path.getsize(self.filename)
942 939 currentSize = self.fileSize - currentPointer
943 940
944 941 if (currentSize >= neededSize):
945 942 self.basicHeaderObj.read(self.fp)
946 943 return 1
947 944
948 945 if self.fileSize == self.fileSizeByHeader:
949 946 # self.flagEoF = True
950 947 return 0
951 948
952 949 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
953 950 sleep(self.delay)
954 951
955 952 return 0
956 953
957 954 def waitDataBlock(self, pointer_location):
958 955
959 956 currentPointer = pointer_location
960 957
961 958 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
962 959
963 960 for nTries in range(self.nTries):
964 961 self.fp.close()
965 962 self.fp = open(self.filename, 'rb')
966 963 self.fp.seek(currentPointer)
967 964
968 965 self.fileSize = os.path.getsize(self.filename)
969 966 currentSize = self.fileSize - currentPointer
970 967
971 968 if (currentSize >= neededSize):
972 969 return 1
973 970
974 971 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1)
975 972 sleep(self.delay)
976 973
977 974 return 0
978 975
979 976 def __jumpToLastBlock(self):
980 977
981 978 if not(self.__isFirstTimeOnline):
982 979 return
983 980
984 981 csize = self.fileSize - self.fp.tell()
985 982 blocksize = self.processingHeaderObj.blockSize
986 983
987 984 # salta el primer bloque de datos
988 985 if csize > self.processingHeaderObj.blockSize:
989 986 self.fp.seek(self.fp.tell() + blocksize)
990 987 else:
991 988 return
992 989
993 990 csize = self.fileSize - self.fp.tell()
994 991 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
995 992 while True:
996 993
997 994 if self.fp.tell() < self.fileSize:
998 995 self.fp.seek(self.fp.tell() + neededsize)
999 996 else:
1000 997 self.fp.seek(self.fp.tell() - neededsize)
1001 998 break
1002 999
1003 1000 # csize = self.fileSize - self.fp.tell()
1004 1001 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1005 1002 # factor = int(csize/neededsize)
1006 1003 # if factor > 0:
1007 1004 # self.fp.seek(self.fp.tell() + factor*neededsize)
1008 1005
1009 1006 self.flagIsNewFile = 0
1010 1007 self.__isFirstTimeOnline = 0
1011 1008
1012 1009 def __setNewBlock(self):
1013 1010 # if self.server is None:
1014 1011 if self.fp == None:
1015 1012 return 0
1016 1013
1017 1014 # if self.online:
1018 1015 # self.__jumpToLastBlock()
1019 1016
1020 1017 if self.flagIsNewFile:
1021 1018 self.lastUTTime = self.basicHeaderObj.utc
1022 1019 return 1
1023 1020
1024 1021 if self.realtime:
1025 1022 self.flagDiscontinuousBlock = 1
1026 1023 if not(self.setNextFile()):
1027 1024 return 0
1028 1025 else:
1029 1026 return 1
1030 1027 # if self.server is None:
1031 1028 currentSize = self.fileSize - self.fp.tell()
1032 1029 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1033 1030 if (currentSize >= neededSize):
1034 1031 self.basicHeaderObj.read(self.fp)
1035 1032 self.lastUTTime = self.basicHeaderObj.utc
1036 1033 return 1
1037 1034 # else:
1038 1035 # self.basicHeaderObj.read(self.zHeader)
1039 1036 # self.lastUTTime = self.basicHeaderObj.utc
1040 1037 # return 1
1041 1038 if self.__waitNewBlock():
1042 1039 self.lastUTTime = self.basicHeaderObj.utc
1043 1040 return 1
1044 1041 # if self.server is None:
1045 1042 if not(self.setNextFile()):
1046 1043 return 0
1047 1044
1048 1045 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1049 1046 self.lastUTTime = self.basicHeaderObj.utc
1050 1047
1051 1048 self.flagDiscontinuousBlock = 0
1052 1049
1053 1050 if deltaTime > self.maxTimeStep:
1054 1051 self.flagDiscontinuousBlock = 1
1055 1052
1056 1053 return 1
1057 1054
1058 1055 def readNextBlock(self):
1059 1056
1060 1057 # Skip block out of startTime and endTime
1061 1058 while True:
1062 1059 if not(self.__setNewBlock()):
1063 1060 return 0
1064 1061
1065 1062 if not(self.readBlock()):
1066 1063 return 0
1067 1064
1068 1065 self.getBasicHeader()
1069
1070 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1071
1066 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1072 1067 print "[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1073 1068 self.processingHeaderObj.dataBlocksPerFile,
1074 1069 self.dataOut.datatime.ctime())
1075 1070 continue
1076 1071
1077 1072 break
1078 1073
1079 1074 if self.verbose:
1080 1075 print "[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1081 1076 self.processingHeaderObj.dataBlocksPerFile,
1082 1077 self.dataOut.datatime.ctime())
1083 1078 return 1
1084 1079
1085 1080 def __readFirstHeader(self):
1086 1081
1087 1082 self.basicHeaderObj.read(self.fp)
1088 1083 self.systemHeaderObj.read(self.fp)
1089 1084 self.radarControllerHeaderObj.read(self.fp)
1090 1085 self.processingHeaderObj.read(self.fp)
1091 1086
1092 1087 self.firstHeaderSize = self.basicHeaderObj.size
1093 1088
1094 1089 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1095 1090 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1096 1091 if datatype == 0:
1097 1092 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1098 1093 elif datatype == 1:
1099 1094 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1100 1095 elif datatype == 2:
1101 1096 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1102 1097 elif datatype == 3:
1103 1098 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1104 1099 elif datatype == 4:
1105 1100 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1106 1101 elif datatype == 5:
1107 1102 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1108 1103 else:
1109 1104 raise ValueError, 'Data type was not defined'
1110 1105
1111 1106 self.dtype = datatype_str
1112 1107 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1113 1108 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1114 1109 self.firstHeaderSize + self.basicHeaderSize * \
1115 1110 (self.processingHeaderObj.dataBlocksPerFile - 1)
1116 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1117 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1111 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1112 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1118 1113 self.getBlockDimension()
1119 1114
1120 1115 def __verifyFile(self, filename, msgFlag=True):
1121 1116
1122 1117 msg = None
1123 1118
1124 1119 try:
1125 1120 fp = open(filename, 'rb')
1126 1121 except IOError:
1127 1122
1128 1123 if msgFlag:
1129 1124 print "[Reading] File %s can't be opened" % (filename)
1130 1125
1131 1126 return False
1132 1127
1133 1128 currentPosition = fp.tell()
1134 1129 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1135 1130
1136 1131 if neededSize == 0:
1137 1132 basicHeaderObj = BasicHeader(LOCALTIME)
1138 1133 systemHeaderObj = SystemHeader()
1139 1134 radarControllerHeaderObj = RadarControllerHeader()
1140 1135 processingHeaderObj = ProcessingHeader()
1141 1136
1142 1137 if not(basicHeaderObj.read(fp)):
1143 1138 fp.close()
1144 1139 return False
1145 1140
1146 1141 if not(systemHeaderObj.read(fp)):
1147 1142 fp.close()
1148 1143 return False
1149 1144
1150 1145 if not(radarControllerHeaderObj.read(fp)):
1151 1146 fp.close()
1152 1147 return False
1153 1148
1154 1149 if not(processingHeaderObj.read(fp)):
1155 1150 fp.close()
1156 1151 return False
1157 1152
1158 1153 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1159 1154 else:
1160 1155 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1161 1156
1162 1157 fp.close()
1163 1158
1164 1159 fileSize = os.path.getsize(filename)
1165 1160 currentSize = fileSize - currentPosition
1166 1161
1167 1162 if currentSize < neededSize:
1168 1163 if msgFlag and (msg != None):
1169 1164 print msg
1170 1165 return False
1171 1166
1172 1167 return True
1173 1168
1174 1169 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1175 1170
1176 1171 path_empty = True
1177 1172
1178 1173 dateList = []
1179 1174 pathList = []
1180 1175
1181 1176 multi_path = path.split(',')
1182 1177
1183 1178 if not walk:
1184 1179
1185 1180 for single_path in multi_path:
1186 1181
1187 1182 if not os.path.isdir(single_path):
1188 1183 continue
1189 1184
1190 1185 fileList = glob.glob1(single_path, "*" + ext)
1191 1186
1192 1187 if not fileList:
1193 1188 continue
1194 1189
1195 1190 path_empty = False
1196 1191
1197 1192 fileList.sort()
1198 1193
1199 1194 for thisFile in fileList:
1200 1195
1201 1196 if not os.path.isfile(os.path.join(single_path, thisFile)):
1202 1197 continue
1203 1198
1204 1199 if not isRadarFile(thisFile):
1205 1200 continue
1206 1201
1207 1202 if not isFileInDateRange(thisFile, startDate, endDate):
1208 1203 continue
1209 1204
1210 1205 thisDate = getDateFromRadarFile(thisFile)
1211 1206
1212 1207 if thisDate in dateList:
1213 1208 continue
1214 1209
1215 1210 dateList.append(thisDate)
1216 1211 pathList.append(single_path)
1217 1212
1218 1213 else:
1219 1214 for single_path in multi_path:
1220 1215
1221 1216 if not os.path.isdir(single_path):
1222 1217 continue
1223 1218
1224 1219 dirList = []
1225 1220
1226 1221 for thisPath in os.listdir(single_path):
1227 1222
1228 1223 if not os.path.isdir(os.path.join(single_path, thisPath)):
1229 1224 continue
1230 1225
1231 1226 if not isRadarFolder(thisPath):
1232 1227 continue
1233 1228
1234 1229 if not isFolderInDateRange(thisPath, startDate, endDate):
1235 1230 continue
1236 1231
1237 1232 dirList.append(thisPath)
1238 1233
1239 1234 if not dirList:
1240 1235 continue
1241 1236
1242 1237 dirList.sort()
1243 1238
1244 1239 for thisDir in dirList:
1245 1240
1246 1241 datapath = os.path.join(single_path, thisDir, expLabel)
1247 1242 fileList = glob.glob1(datapath, "*" + ext)
1248 1243
1249 1244 if not fileList:
1250 1245 continue
1251 1246
1252 1247 path_empty = False
1253 1248
1254 1249 thisDate = getDateFromRadarFolder(thisDir)
1255 1250
1256 1251 pathList.append(datapath)
1257 1252 dateList.append(thisDate)
1258 1253
1259 1254 dateList.sort()
1260 1255
1261 1256 if walk:
1262 1257 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1263 1258 else:
1264 1259 pattern_path = multi_path[0]
1265 1260
1266 1261 if path_empty:
1267 1262 print "[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate)
1268 1263 else:
1269 1264 if not dateList:
1270 1265 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path)
1271 1266
1272 1267 if include_path:
1273 1268 return dateList, pathList
1274 1269
1275 1270 return dateList
1276 1271
1277 1272 def setup(self,
1278 1273 path=None,
1279 1274 startDate=None,
1280 1275 endDate=None,
1281 1276 startTime=datetime.time(0, 0, 0),
1282 1277 endTime=datetime.time(23, 59, 59),
1283 1278 set=None,
1284 1279 expLabel="",
1285 1280 ext=None,
1286 1281 online=False,
1287 1282 delay=60,
1288 1283 walk=True,
1289 1284 getblock=False,
1290 1285 nTxs=1,
1291 1286 realtime=False,
1292 1287 blocksize=None,
1293 1288 blocktime=None,
1294 1289 skip=None,
1295 1290 cursor=None,
1296 1291 warnings=True,
1297 1292 verbose=True,
1298 1293 server=None,
1299 1294 format=None,
1300 1295 oneDDict=None,
1301 1296 twoDDict=None,
1302 1297 ind2DList=None):
1303 1298 if server is not None:
1304 1299 if 'tcp://' in server:
1305 1300 address = server
1306 1301 else:
1307 1302 address = 'ipc:///tmp/%s' % server
1308 1303 self.server = address
1309 1304 self.context = zmq.Context()
1310 1305 self.receiver = self.context.socket(zmq.PULL)
1311 1306 self.receiver.connect(self.server)
1312 1307 time.sleep(0.5)
1313 1308 print '[Starting] ReceiverData from {}'.format(self.server)
1314 1309 else:
1315 1310 self.server = None
1316 1311 if path == None:
1317 1312 raise ValueError, "[Reading] The path is not valid"
1318 1313
1319 1314 if ext == None:
1320 1315 ext = self.ext
1321 1316
1322 1317 if online:
1323 1318 print "[Reading] Searching files in online mode..."
1324 1319
1325 1320 for nTries in range(self.nTries):
1326 1321 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1327 1322 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1328 1323
1329 1324 if fullpath:
1330 1325 break
1331 1326
1332 1327 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1)
1333 1328 sleep(self.delay)
1334 1329
1335 1330 if not(fullpath):
1336 1331 print "[Reading] There 'isn't any valid file in %s" % path
1337 1332 return
1338 1333
1339 1334 self.year = year
1340 1335 self.doy = doy
1341 1336 self.set = set - 1
1342 1337 self.path = path
1343 1338 self.foldercounter = foldercounter
1344 1339 last_set = None
1345 1340 else:
1346 1341 print "[Reading] Searching files in offline mode ..."
1347 1342 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1348 1343 startTime=startTime, endTime=endTime,
1349 1344 set=set, expLabel=expLabel, ext=ext,
1350 1345 walk=walk, cursor=cursor,
1351 1346 skip=skip)
1352 1347
1353 1348 if not(pathList):
1354 1349 self.fileIndex = -1
1355 1350 self.pathList = []
1356 1351 self.filenameList = []
1357 1352 return
1358 1353
1359 1354 self.fileIndex = -1
1360 1355 self.pathList = pathList
1361 1356 self.filenameList = filenameList
1362 1357 file_name = os.path.basename(filenameList[-1])
1363 1358 basename, ext = os.path.splitext(file_name)
1364 1359 last_set = int(basename[-3:])
1365 1360
1366 1361 self.online = online
1367 1362 self.realtime = realtime
1368 1363 self.delay = delay
1369 1364 ext = ext.lower()
1370 1365 self.ext = ext
1371 1366 self.getByBlock = getblock
1372 1367 self.nTxs = nTxs
1373 1368 self.startTime = startTime
1374 1369 self.endTime = endTime
1375
1370 self.endDate = endDate
1371 self.startDate = startDate
1376 1372 # Added-----------------
1377 1373 self.selBlocksize = blocksize
1378 1374 self.selBlocktime = blocktime
1379 1375
1380 1376 # Verbose-----------
1381 1377 self.verbose = verbose
1382 1378 self.warnings = warnings
1383 1379
1384 1380 if not(self.setNextFile()):
1385 1381 if (startDate != None) and (endDate != None):
1386 1382 print "[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime())
1387 1383 elif startDate != None:
1388 1384 print "[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime())
1389 1385 else:
1390 1386 print "[Reading] No files"
1391 1387
1392 1388 self.fileIndex = -1
1393 1389 self.pathList = []
1394 1390 self.filenameList = []
1395 1391 return
1396 1392
1397 1393 # self.getBasicHeader()
1398 1394
1399 1395 if last_set != None:
1400 1396 self.dataOut.last_block = last_set * \
1401 1397 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1402 1398 return
1403 1399
1404 1400 def getBasicHeader(self):
1405 1401
1406 1402 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1407 1403 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1408 1404
1409 1405 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1410 1406
1411 1407 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1412 1408
1413 1409 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1414 1410
1415 1411 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1416 1412
1417 1413 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1418 1414
1419 1415 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1420 1416
1421 1417 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1422 1418
1423 1419 def getFirstHeader(self):
1424 1420
1425 1421 raise NotImplementedError
1426 1422
1427 1423 def getData(self):
1428 1424
1429 1425 raise NotImplementedError
1430 1426
1431 1427 def hasNotDataInBuffer(self):
1432 1428
1433 1429 raise NotImplementedError
1434 1430
1435 1431 def readBlock(self):
1436 1432
1437 1433 raise NotImplementedError
1438 1434
1439 1435 def isEndProcess(self):
1440 1436
1441 1437 return self.flagNoMoreFiles
1442 1438
1443 1439 def printReadBlocks(self):
1444 1440
1445 1441 print "[Reading] Number of read blocks per file %04d" % self.nReadBlocks
1446 1442
1447 1443 def printTotalBlocks(self):
1448 1444
1449 1445 print "[Reading] Number of read blocks %04d" % self.nTotalBlocks
1450 1446
1451 1447 def printNumberOfBlock(self):
1452 1448 'SPAM!'
1453 1449
1454 1450 # if self.flagIsNewBlock:
1455 1451 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1456 1452 # self.processingHeaderObj.dataBlocksPerFile,
1457 1453 # self.dataOut.datatime.ctime())
1458 1454
1459 1455 def printInfo(self):
1460 1456
1461 1457 if self.__printInfo == False:
1462 1458 return
1463 1459
1464 1460 self.basicHeaderObj.printInfo()
1465 1461 self.systemHeaderObj.printInfo()
1466 1462 self.radarControllerHeaderObj.printInfo()
1467 1463 self.processingHeaderObj.printInfo()
1468 1464
1469 1465 self.__printInfo = False
1470 1466
1471 1467 def run(self,
1472 1468 path=None,
1473 1469 startDate=None,
1474 1470 endDate=None,
1475 1471 startTime=datetime.time(0, 0, 0),
1476 1472 endTime=datetime.time(23, 59, 59),
1477 1473 set=None,
1478 1474 expLabel="",
1479 1475 ext=None,
1480 1476 online=False,
1481 1477 delay=60,
1482 1478 walk=True,
1483 1479 getblock=False,
1484 1480 nTxs=1,
1485 1481 realtime=False,
1486 1482 blocksize=None,
1487 1483 blocktime=None,
1488 1484 skip=None,
1489 1485 cursor=None,
1490 1486 warnings=True,
1491 1487 server=None,
1492 1488 verbose=True,
1493 1489 format=None,
1494 1490 oneDDict=None,
1495 1491 twoDDict=None,
1496 1492 ind2DList=None, **kwargs):
1497 1493
1498 1494 if not(self.isConfig):
1499 1495 self.setup(path=path,
1500 1496 startDate=startDate,
1501 1497 endDate=endDate,
1502 1498 startTime=startTime,
1503 1499 endTime=endTime,
1504 1500 set=set,
1505 1501 expLabel=expLabel,
1506 1502 ext=ext,
1507 1503 online=online,
1508 1504 delay=delay,
1509 1505 walk=walk,
1510 1506 getblock=getblock,
1511 1507 nTxs=nTxs,
1512 1508 realtime=realtime,
1513 1509 blocksize=blocksize,
1514 1510 blocktime=blocktime,
1515 1511 skip=skip,
1516 1512 cursor=cursor,
1517 1513 warnings=warnings,
1518 1514 server=server,
1519 1515 verbose=verbose,
1520 1516 format=format,
1521 1517 oneDDict=oneDDict,
1522 1518 twoDDict=twoDDict,
1523 1519 ind2DList=ind2DList)
1524 1520 self.isConfig = True
1525 1521 if server is None:
1526 1522 self.getData()
1527 1523 else:
1528 1524 self.getFromServer()
1529 1525
1530 1526
1531 1527 class JRODataWriter(JRODataIO):
1532 1528
1533 1529 """
1534 1530 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1535 1531 de los datos siempre se realiza por bloques.
1536 1532 """
1537 1533
1538 1534 blockIndex = 0
1539 1535
1540 1536 path = None
1541 1537
1542 1538 setFile = None
1543 1539
1544 1540 profilesPerBlock = None
1545 1541
1546 1542 blocksPerFile = None
1547 1543
1548 1544 nWriteBlocks = 0
1549 1545
1550 1546 fileDate = None
1551 1547
1552 1548 def __init__(self, dataOut=None):
1553 1549 raise NotImplementedError
1554 1550
1555 1551 def hasAllDataInBuffer(self):
1556 1552 raise NotImplementedError
1557 1553
1558 1554 def setBlockDimension(self):
1559 1555 raise NotImplementedError
1560 1556
1561 1557 def writeBlock(self):
1562 1558 raise NotImplementedError
1563 1559
1564 1560 def putData(self):
1565 1561 raise NotImplementedError
1566 1562
1567 1563 def getProcessFlags(self):
1568 1564
1569 1565 processFlags = 0
1570 1566
1571 1567 dtype_index = get_dtype_index(self.dtype)
1572 1568 procflag_dtype = get_procflag_dtype(dtype_index)
1573 1569
1574 1570 processFlags += procflag_dtype
1575 1571
1576 1572 if self.dataOut.flagDecodeData:
1577 1573 processFlags += PROCFLAG.DECODE_DATA
1578 1574
1579 1575 if self.dataOut.flagDeflipData:
1580 1576 processFlags += PROCFLAG.DEFLIP_DATA
1581 1577
1582 1578 if self.dataOut.code is not None:
1583 1579 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1584 1580
1585 1581 if self.dataOut.nCohInt > 1:
1586 1582 processFlags += PROCFLAG.COHERENT_INTEGRATION
1587 1583
1588 1584 if self.dataOut.type == "Spectra":
1589 1585 if self.dataOut.nIncohInt > 1:
1590 1586 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1591 1587
1592 1588 if self.dataOut.data_dc is not None:
1593 1589 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1594 1590
1595 1591 if self.dataOut.flagShiftFFT:
1596 1592 processFlags += PROCFLAG.SHIFT_FFT_DATA
1597 1593
1598 1594 return processFlags
1599 1595
1600 1596 def setBasicHeader(self):
1601 1597
1602 1598 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1603 1599 self.basicHeaderObj.version = self.versionFile
1604 1600 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1605 1601
1606 1602 utc = numpy.floor(self.dataOut.utctime)
1607 1603 milisecond = (self.dataOut.utctime - utc) * 1000.0
1608 1604
1609 1605 self.basicHeaderObj.utc = utc
1610 1606 self.basicHeaderObj.miliSecond = milisecond
1611 1607 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1612 1608 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1613 1609 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1614 1610
1615 1611 def setFirstHeader(self):
1616 1612 """
1617 1613 Obtiene una copia del First Header
1618 1614
1619 1615 Affected:
1620 1616
1621 1617 self.basicHeaderObj
1622 1618 self.systemHeaderObj
1623 1619 self.radarControllerHeaderObj
1624 1620 self.processingHeaderObj self.
1625 1621
1626 1622 Return:
1627 1623 None
1628 1624 """
1629 1625
1630 1626 raise NotImplementedError
1631 1627
1632 1628 def __writeFirstHeader(self):
1633 1629 """
1634 1630 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1635 1631
1636 1632 Affected:
1637 1633 __dataType
1638 1634
1639 1635 Return:
1640 1636 None
1641 1637 """
1642 1638
1643 1639 # CALCULAR PARAMETROS
1644 1640
1645 1641 sizeLongHeader = self.systemHeaderObj.size + \
1646 1642 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1647 1643 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1648 1644
1649 1645 self.basicHeaderObj.write(self.fp)
1650 1646 self.systemHeaderObj.write(self.fp)
1651 1647 self.radarControllerHeaderObj.write(self.fp)
1652 1648 self.processingHeaderObj.write(self.fp)
1653 1649
1654 1650 def __setNewBlock(self):
1655 1651 """
1656 1652 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1657 1653
1658 1654 Return:
1659 1655 0 : si no pudo escribir nada
1660 1656 1 : Si escribio el Basic el First Header
1661 1657 """
1662 1658 if self.fp == None:
1663 1659 self.setNextFile()
1664 1660
1665 1661 if self.flagIsNewFile:
1666 1662 return 1
1667 1663
1668 1664 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1669 1665 self.basicHeaderObj.write(self.fp)
1670 1666 return 1
1671 1667
1672 1668 if not(self.setNextFile()):
1673 1669 return 0
1674 1670
1675 1671 return 1
1676 1672
1677 1673 def writeNextBlock(self):
1678 1674 """
1679 1675 Selecciona el bloque siguiente de datos y los escribe en un file
1680 1676
1681 1677 Return:
1682 1678 0 : Si no hizo pudo escribir el bloque de datos
1683 1679 1 : Si no pudo escribir el bloque de datos
1684 1680 """
1685 1681 if not(self.__setNewBlock()):
1686 1682 return 0
1687 1683
1688 1684 self.writeBlock()
1689 1685
1690 1686 print "[Writing] Block No. %d/%d" % (self.blockIndex,
1691 1687 self.processingHeaderObj.dataBlocksPerFile)
1692 1688
1693 1689 return 1
1694 1690
1695 1691 def setNextFile(self):
1696 1692 """
1697 1693 Determina el siguiente file que sera escrito
1698 1694
1699 1695 Affected:
1700 1696 self.filename
1701 1697 self.subfolder
1702 1698 self.fp
1703 1699 self.setFile
1704 1700 self.flagIsNewFile
1705 1701
1706 1702 Return:
1707 1703 0 : Si el archivo no puede ser escrito
1708 1704 1 : Si el archivo esta listo para ser escrito
1709 1705 """
1710 1706 ext = self.ext
1711 1707 path = self.path
1712 1708
1713 1709 if self.fp != None:
1714 1710 self.fp.close()
1715 1711
1716 1712 timeTuple = time.localtime(self.dataOut.utctime)
1717 1713 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1718 1714
1719 1715 fullpath = os.path.join(path, subfolder)
1720 1716 setFile = self.setFile
1721 1717
1722 1718 if not(os.path.exists(fullpath)):
1723 1719 os.mkdir(fullpath)
1724 1720 setFile = -1 # inicializo mi contador de seteo
1725 1721 else:
1726 1722 filesList = os.listdir(fullpath)
1727 1723 if len(filesList) > 0:
1728 1724 filesList = sorted(filesList, key=str.lower)
1729 1725 filen = filesList[-1]
1730 1726 # el filename debera tener el siguiente formato
1731 1727 # 0 1234 567 89A BCDE (hex)
1732 1728 # x YYYY DDD SSS .ext
1733 1729 if isNumber(filen[8:11]):
1734 1730 # inicializo mi contador de seteo al seteo del ultimo file
1735 1731 setFile = int(filen[8:11])
1736 1732 else:
1737 1733 setFile = -1
1738 1734 else:
1739 1735 setFile = -1 # inicializo mi contador de seteo
1740 1736
1741 1737 setFile += 1
1742 1738
1743 1739 # If this is a new day it resets some values
1744 1740 if self.dataOut.datatime.date() > self.fileDate:
1745 1741 setFile = 0
1746 1742 self.nTotalBlocks = 0
1747 1743
1748 1744 filen = '%s%4.4d%3.3d%3.3d%s' % (
1749 1745 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1750 1746
1751 1747 filename = os.path.join(path, subfolder, filen)
1752 1748
1753 1749 fp = open(filename, 'wb')
1754 1750
1755 1751 self.blockIndex = 0
1756 1752
1757 1753 # guardando atributos
1758 1754 self.filename = filename
1759 1755 self.subfolder = subfolder
1760 1756 self.fp = fp
1761 1757 self.setFile = setFile
1762 1758 self.flagIsNewFile = 1
1763 1759 self.fileDate = self.dataOut.datatime.date()
1764 1760
1765 1761 self.setFirstHeader()
1766 1762
1767 1763 print '[Writing] Opening file: %s' % self.filename
1768 1764
1769 1765 self.__writeFirstHeader()
1770 1766
1771 1767 return 1
1772 1768
1773 1769 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1774 1770 """
1775 1771 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1776 1772
1777 1773 Inputs:
1778 1774 path : directory where data will be saved
1779 1775 profilesPerBlock : number of profiles per block
1780 1776 set : initial file set
1781 1777 datatype : An integer number that defines data type:
1782 1778 0 : int8 (1 byte)
1783 1779 1 : int16 (2 bytes)
1784 1780 2 : int32 (4 bytes)
1785 1781 3 : int64 (8 bytes)
1786 1782 4 : float32 (4 bytes)
1787 1783 5 : double64 (8 bytes)
1788 1784
1789 1785 Return:
1790 1786 0 : Si no realizo un buen seteo
1791 1787 1 : Si realizo un buen seteo
1792 1788 """
1793 1789
1794 1790 if ext == None:
1795 1791 ext = self.ext
1796 1792
1797 1793 self.ext = ext.lower()
1798 1794
1799 1795 self.path = path
1800 1796
1801 1797 if set is None:
1802 1798 self.setFile = -1
1803 1799 else:
1804 1800 self.setFile = set - 1
1805 1801
1806 1802 self.blocksPerFile = blocksPerFile
1807 1803
1808 1804 self.profilesPerBlock = profilesPerBlock
1809 1805
1810 1806 self.dataOut = dataOut
1811 1807 self.fileDate = self.dataOut.datatime.date()
1812 1808 # By default
1813 1809 self.dtype = self.dataOut.dtype
1814 1810
1815 1811 if datatype is not None:
1816 1812 self.dtype = get_numpy_dtype(datatype)
1817 1813
1818 1814 if not(self.setNextFile()):
1819 1815 print "[Writing] There isn't a next file"
1820 1816 return 0
1821 1817
1822 1818 self.setBlockDimension()
1823 1819
1824 1820 return 1
1825 1821
1826 1822 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1827 1823
1828 1824 if not(self.isConfig):
1829 1825
1830 1826 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1831 1827 set=set, ext=ext, datatype=datatype, **kwargs)
1832 1828 self.isConfig = True
1833 1829
1834 1830 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now