##// END OF EJS Templates
schain
RSS
Bug fixed in jroIO_base:...
Miguel Valdez -
r632:3ab62b8f427e
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1529 +1,1550
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 traceback.print_exc()
72 72 raise IOError, "The file %s can't be opened" %(filename)
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isFileinThisTime(filename, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
94 94
95 95 endTime : tiempo final del rango seleccionado en formato datetime.time
96 96
97 97 Return:
98 98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
99 99 fecha especificado, de lo contrario retorna False.
100 100
101 101 Excepciones:
102 102 Si el archivo no existe o no puede ser abierto
103 103 Si la cabecera no puede ser leida.
104 104
105 105 """
106 106
107 107
108 108 try:
109 109 fp = open(filename,'rb')
110 110 except IOError:
111 111 traceback.print_exc()
112 112 raise IOError, "The file %s can't be opened" %(filename)
113 113
114 114 basicHeaderObj = BasicHeader(LOCALTIME)
115 115 sts = basicHeaderObj.read(fp)
116 116 fp.close()
117 117
118 118 thisDatetime = basicHeaderObj.datatime
119 119 thisTime = thisDatetime.time()
120 120
121 121 if not(sts):
122 122 print "Skipping the file %s because it has not a valid header" %(filename)
123 123 return None
124 124
125 125 if not ((startTime <= thisTime) and (endTime > thisTime)):
126 126 return None
127 127
128 128 return thisDatetime
129 129
130 130 def getFileFromSet(path, ext, set):
131 131 validFilelist = []
132 132 fileList = os.listdir(path)
133 133
134 134 # 0 1234 567 89A BCDE
135 135 # H YYYY DDD SSS .ext
136 136
137 137 for thisFile in fileList:
138 138 try:
139 139 year = int(thisFile[1:5])
140 140 doy = int(thisFile[5:8])
141 141 except:
142 142 continue
143 143
144 144 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
145 145 continue
146 146
147 147 validFilelist.append(thisFile)
148 148
149 149 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
150 150
151 151 if len(myfile)!= 0:
152 152 return myfile[0]
153 153 else:
154 154 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
155 155 print 'the filename %s does not exist'%filename
156 156 print '...going to the last file: '
157 157
158 158 if validFilelist:
159 159 validFilelist = sorted( validFilelist, key=str.lower )
160 160 return validFilelist[-1]
161 161
162 162 return None
163 163
164 164 def getlastFileFromPath(path, ext):
165 165 """
166 166 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
167 167 al final de la depuracion devuelve el ultimo file de la lista que quedo.
168 168
169 169 Input:
170 170 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
171 171 ext : extension de los files contenidos en una carpeta
172 172
173 173 Return:
174 174 El ultimo file de una determinada carpeta, no se considera el path.
175 175 """
176 176 validFilelist = []
177 177 fileList = os.listdir(path)
178 178
179 179 # 0 1234 567 89A BCDE
180 180 # H YYYY DDD SSS .ext
181 181
182 182 for thisFile in fileList:
183 183
184 184 year = thisFile[1:5]
185 185 if not isNumber(year):
186 186 continue
187 187
188 188 doy = thisFile[5:8]
189 189 if not isNumber(doy):
190 190 continue
191 191
192 192 year = int(year)
193 193 doy = int(doy)
194 194
195 195 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
196 196 continue
197 197
198 198 validFilelist.append(thisFile)
199 199
200 200 if validFilelist:
201 201 validFilelist = sorted( validFilelist, key=str.lower )
202 202 return validFilelist[-1]
203 203
204 204 return None
205 205
206 206 def checkForRealPath(path, foldercounter, year, doy, set, ext):
207 207 """
208 208 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
209 209 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
210 210 el path exacto de un determinado file.
211 211
212 212 Example :
213 213 nombre correcto del file es .../.../D2009307/P2009307367.ext
214 214
215 215 Entonces la funcion prueba con las siguientes combinaciones
216 216 .../.../y2009307367.ext
217 217 .../.../Y2009307367.ext
218 218 .../.../x2009307/y2009307367.ext
219 219 .../.../x2009307/Y2009307367.ext
220 220 .../.../X2009307/y2009307367.ext
221 221 .../.../X2009307/Y2009307367.ext
222 222 siendo para este caso, la ultima combinacion de letras, identica al file buscado
223 223
224 224 Return:
225 225 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
226 226 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
227 227 para el filename
228 228 """
229 229 fullfilename = None
230 230 find_flag = False
231 231 filename = None
232 232
233 233 prefixDirList = [None,'d','D']
234 234 if ext.lower() == ".r": #voltage
235 235 prefixFileList = ['d','D']
236 236 elif ext.lower() == ".pdata": #spectra
237 237 prefixFileList = ['p','P']
238 238 else:
239 239 return None, filename
240 240
241 241 #barrido por las combinaciones posibles
242 242 for prefixDir in prefixDirList:
243 243 thispath = path
244 244 if prefixDir != None:
245 245 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
246 246 if foldercounter == 0:
247 247 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
248 248 else:
249 249 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
250 250 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
251 251 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
252 252 fullfilename = os.path.join( thispath, filename ) #formo el path completo
253 253
254 254 if os.path.exists( fullfilename ): #verifico que exista
255 255 find_flag = True
256 256 break
257 257 if find_flag:
258 258 break
259 259
260 260 if not(find_flag):
261 261 return None, filename
262 262
263 263 return fullfilename, filename
264 264
265 265 def isRadarFolder(folder):
266 266 try:
267 267 year = int(folder[1:5])
268 268 doy = int(folder[5:8])
269 269 except:
270 270 return 0
271 271
272 272 return 1
273 273
274 274 def isRadarFile(file):
275 275 try:
276 276 year = int(file[1:5])
277 277 doy = int(file[5:8])
278 278 set = int(file[8:11])
279 279 except:
280 280 return 0
281 281
282 282 return 1
283 283
284 284 def getDateFromRadarFile(file):
285 285 try:
286 286 year = int(file[1:5])
287 287 doy = int(file[5:8])
288 288 set = int(file[8:11])
289 289 except:
290 290 return None
291 291
292 292 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
293 293 return thisDate
294 294
295 295 class JRODataIO:
296 296
297 297 c = 3E8
298 298
299 299 isConfig = False
300 300
301 301 basicHeaderObj = None
302 302
303 303 systemHeaderObj = None
304 304
305 305 radarControllerHeaderObj = None
306 306
307 307 processingHeaderObj = None
308 308
309 309 online = 0
310 310
311 311 dtype = None
312 312
313 313 pathList = []
314 314
315 315 filenameList = []
316 316
317 317 filename = None
318 318
319 319 ext = None
320 320
321 321 flagIsNewFile = 1
322 322
323 323 flagDiscontinuousBlock = 0
324 324
325 325 flagIsNewBlock = 0
326 326
327 327 fp = None
328 328
329 329 firstHeaderSize = 0
330 330
331 331 basicHeaderSize = 24
332 332
333 333 versionFile = 1103
334 334
335 335 fileSize = None
336 336
337 337 # ippSeconds = None
338 338
339 339 fileSizeByHeader = None
340 340
341 341 fileIndex = None
342 342
343 343 profileIndex = None
344 344
345 345 blockIndex = None
346 346
347 347 nTotalBlocks = None
348 348
349 349 maxTimeStep = 30
350 350
351 351 lastUTTime = None
352 352
353 353 datablock = None
354 354
355 355 dataOut = None
356 356
357 357 blocksize = None
358 358
359 359 getByBlock = False
360 360
361 361 def __init__(self):
362 362
363 363 raise ValueError, "Not implemented"
364 364
365 365 def run(self):
366 366
367 367 raise ValueError, "Not implemented"
368 368
369 369 def getDtypeWidth(self):
370 370
371 371 dtype_index = get_dtype_index(self.dtype)
372 372 dtype_width = get_dtype_width(dtype_index)
373 373
374 374 return dtype_width
375 375
376 376 class JRODataReader(JRODataIO):
377 377
378 378 nReadBlocks = 0
379 379
380 380 delay = 10 #number of seconds waiting a new file
381 381
382 382 nTries = 3 #quantity tries
383 383
384 384 nFiles = 3 #number of files for searching
385 385
386 386 path = None
387 387
388 388 foldercounter = 0
389 389
390 390 flagNoMoreFiles = 0
391 391
392 392 datetimeList = []
393 393
394 394 __isFirstTimeOnline = 1
395 395
396 396 __printInfo = True
397 397
398 398 profileIndex = None
399 399
400 400 nTxs = 1
401 401
402 402 txIndex = None
403 403
404 404 def __init__(self):
405 405
406 406 """
407 407
408 408 """
409 409
410 410 # raise NotImplementedError, "This method has not been implemented"
411 411
412 412
413 413 def createObjByDefault(self):
414 414 """
415 415
416 416 """
417 417 raise NotImplementedError, "This method has not been implemented"
418 418
419 419 def getBlockDimension(self):
420 420
421 421 raise NotImplementedError, "No implemented"
422 422
423 423 def __searchFilesOffLine(self,
424 424 path,
425 425 startDate=None,
426 426 endDate=None,
427 427 startTime=datetime.time(0,0,0),
428 428 endTime=datetime.time(23,59,59),
429 429 set=None,
430 430 expLabel='',
431 431 ext='.r',
432 432 walk=True):
433 433
434 434 self.filenameList = []
435 435 self.datetimeList = []
436 436
437 437 pathList = []
438 438
439 439 if not walk:
440 440 #pathList.append(path)
441 441 multi_path = path.split(',')
442 442 for single_path in multi_path:
443 443
444 444 if not os.path.isdir(single_path):
445 445 continue
446 446
447 447 pathList.append(single_path)
448 448
449 449 else:
450 450 #dirList = []
451 451 multi_path = path.split(',')
452 452 for single_path in multi_path:
453 453
454 454 if not os.path.isdir(single_path):
455 455 continue
456 456
457 457 dirList = []
458 458 for thisPath in os.listdir(single_path):
459 459 if not os.path.isdir(os.path.join(single_path,thisPath)):
460 460 continue
461 461 if not isRadarFolder(thisPath):
462 462 continue
463 463
464 464 dirList.append(thisPath)
465 465
466 466 if not(dirList):
467 467 return None, None
468 468
469 469 if startDate and endDate:
470 470 thisDate = startDate
471 471
472 472 while(thisDate <= endDate):
473 473 year = thisDate.timetuple().tm_year
474 474 doy = thisDate.timetuple().tm_yday
475 475
476 476 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
477 477 if len(matchlist) == 0:
478 478 thisDate += datetime.timedelta(1)
479 479 continue
480 480 for match in matchlist:
481 481 pathList.append(os.path.join(single_path,match,expLabel))
482 482
483 483 thisDate += datetime.timedelta(1)
484 484 else:
485 485 for thiDir in dirList:
486 486 pathList.append(os.path.join(single_path,thiDir,expLabel))
487 487
488 488 if pathList == []:
489 489 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
490 490 return None, None
491 491
492 492 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
493 493
494 494 filenameList = []
495 495 datetimeList = []
496 496 pathDict = {}
497 497 filenameList_to_sort = []
498 498
499 499 for i in range(len(pathList)):
500 500
501 501 thisPath = pathList[i]
502 502
503 503 fileList = glob.glob1(thisPath, "*%s" %ext)
504 504 if len(fileList) < 1:
505 505 continue
506 506 fileList.sort()
507 507 pathDict.setdefault(fileList[0])
508 508 pathDict[fileList[0]] = i
509 509 filenameList_to_sort.append(fileList[0])
510 510
511 511 filenameList_to_sort.sort()
512 512
513 513 for file in filenameList_to_sort:
514 514 thisPath = pathList[pathDict[file]]
515 515
516 516 fileList = glob.glob1(thisPath, "*%s" %ext)
517 517 fileList.sort()
518 518
519 519 for file in fileList:
520 520
521 521 filename = os.path.join(thisPath,file)
522 522 thisDatetime = isFileinThisTime(filename, startTime, endTime)
523 523
524 524 if not(thisDatetime):
525 525 continue
526 526
527 527 filenameList.append(filename)
528 528 datetimeList.append(thisDatetime)
529 529
530 530 if not(filenameList):
531 531 print "Any file was found for the time range %s - %s" %(startTime, endTime)
532 532 return None, None
533 533
534 534 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
535 535 print
536 536
537 537 for i in range(len(filenameList)):
538 538 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
539 539
540 540 self.filenameList = filenameList
541 541 self.datetimeList = datetimeList
542 542
543 543 return pathList, filenameList
544 544
545 545 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
546 546
547 547 """
548 548 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
549 549 devuelve el archivo encontrado ademas de otros datos.
550 550
551 551 Input:
552 552 path : carpeta donde estan contenidos los files que contiene data
553 553
554 554 expLabel : Nombre del subexperimento (subfolder)
555 555
556 556 ext : extension de los files
557 557
558 558 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
559 559
560 560 Return:
561 561 directory : eL directorio donde esta el file encontrado
562 562 filename : el ultimo file de una determinada carpeta
563 563 year : el anho
564 564 doy : el numero de dia del anho
565 565 set : el set del archivo
566 566
567 567
568 568 """
569 569 dirList = []
570 570
571 571 if not walk:
572 572 fullpath = path
573 573 foldercounter = 0
574 574 else:
575 575 #Filtra solo los directorios
576 576 for thisPath in os.listdir(path):
577 577 if not os.path.isdir(os.path.join(path,thisPath)):
578 578 continue
579 579 if not isRadarFolder(thisPath):
580 580 continue
581 581
582 582 dirList.append(thisPath)
583 583
584 584 if not(dirList):
585 585 return None, None, None, None, None, None
586 586
587 587 dirList = sorted( dirList, key=str.lower )
588 588
589 589 doypath = dirList[-1]
590 590 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
591 591 fullpath = os.path.join(path, doypath, expLabel)
592 592
593 593
594 594 print "%s folder was found: " %(fullpath )
595 595
596 596 if set == None:
597 597 filename = getlastFileFromPath(fullpath, ext)
598 598 else:
599 599 filename = getFileFromSet(fullpath, ext, set)
600 600
601 601 if not(filename):
602 602 return None, None, None, None, None, None
603 603
604 604 print "%s file was found" %(filename)
605 605
606 606 if not(self.__verifyFile(os.path.join(fullpath, filename))):
607 607 return None, None, None, None, None, None
608 608
609 609 year = int( filename[1:5] )
610 610 doy = int( filename[5:8] )
611 611 set = int( filename[8:11] )
612 612
613 613 return fullpath, foldercounter, filename, year, doy, set
614 614
615 615 def __setNextFileOffline(self):
616 616
617 617 idFile = self.fileIndex
618 618
619 619 while (True):
620 620 idFile += 1
621 621 if not(idFile < len(self.filenameList)):
622 622 self.flagNoMoreFiles = 1
623 623 # print "[Reading] No more Files"
624 624 return 0
625 625
626 626 filename = self.filenameList[idFile]
627 627
628 628 if not(self.__verifyFile(filename)):
629 629 continue
630 630
631 631 fileSize = os.path.getsize(filename)
632 632 fp = open(filename,'rb')
633 633 break
634 634
635 635 self.flagIsNewFile = 1
636 636 self.fileIndex = idFile
637 637 self.filename = filename
638 638 self.fileSize = fileSize
639 639 self.fp = fp
640 640
641 641 # print "[Reading] Setting the file: %s"%self.filename
642 642
643 643 return 1
644 644
645 645 def __setNextFileOnline(self):
646 646 """
647 647 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
648 648 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
649 649 siguientes.
650 650
651 651 Affected:
652 652 self.flagIsNewFile
653 653 self.filename
654 654 self.fileSize
655 655 self.fp
656 656 self.set
657 657 self.flagNoMoreFiles
658 658
659 659 Return:
660 660 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
661 661 1 : si el file fue abierto con exito y esta listo a ser leido
662 662
663 663 Excepciones:
664 664 Si un determinado file no puede ser abierto
665 665 """
666 666 nFiles = 0
667 667 fileOk_flag = False
668 668 firstTime_flag = True
669 669
670 670 self.set += 1
671 671
672 672 if self.set > 999:
673 673 self.set = 0
674 674 self.foldercounter += 1
675 675
676 676 #busca el 1er file disponible
677 677 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
678 678 if fullfilename:
679 679 if self.__verifyFile(fullfilename, False):
680 680 fileOk_flag = True
681 681
682 682 #si no encuentra un file entonces espera y vuelve a buscar
683 683 if not(fileOk_flag):
684 684 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
685 685
686 686 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
687 687 tries = self.nTries
688 688 else:
689 689 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
690 690
691 691 for nTries in range( tries ):
692 692 if firstTime_flag:
693 693 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
694 694 sleep( self.delay )
695 695 else:
696 696 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
697 697
698 698 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
699 699 if fullfilename:
700 700 if self.__verifyFile(fullfilename):
701 701 fileOk_flag = True
702 702 break
703 703
704 704 if fileOk_flag:
705 705 break
706 706
707 707 firstTime_flag = False
708 708
709 709 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
710 710 self.set += 1
711 711
712 712 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
713 713 self.set = 0
714 714 self.doy += 1
715 715 self.foldercounter = 0
716 716
717 717 if fileOk_flag:
718 718 self.fileSize = os.path.getsize( fullfilename )
719 719 self.filename = fullfilename
720 720 self.flagIsNewFile = 1
721 721 if self.fp != None: self.fp.close()
722 722 self.fp = open(fullfilename, 'rb')
723 723 self.flagNoMoreFiles = 0
724 724 # print '[Reading] Setting the file: %s' % fullfilename
725 725 else:
726 726 self.fileSize = 0
727 727 self.filename = None
728 728 self.flagIsNewFile = 0
729 729 self.fp = None
730 730 self.flagNoMoreFiles = 1
731 731 # print '[Reading] No more files to read'
732 732
733 733 return fileOk_flag
734 734
735 735 def setNextFile(self):
736 736 if self.fp != None:
737 737 self.fp.close()
738 738
739 739 if self.online:
740 740 newFile = self.__setNextFileOnline()
741 741 else:
742 742 newFile = self.__setNextFileOffline()
743 743
744 744 if not(newFile):
745 745 print '[Reading] No more files to read'
746 746 return 0
747 747
748 748 print '[Reading] Setting the file: %s' % self.filename
749 749
750 750 self.__readFirstHeader()
751 751 self.nReadBlocks = 0
752 752 return 1
753 753
754 754 def __waitNewBlock(self):
755 755 """
756 756 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
757 757
758 758 Si el modo de lectura es OffLine siempre retorn 0
759 759 """
760 760 if not self.online:
761 761 return 0
762 762
763 763 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
764 764 return 0
765 765
766 766 currentPointer = self.fp.tell()
767 767
768 768 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
769 769
770 770 for nTries in range( self.nTries ):
771 771
772 772 self.fp.close()
773 773 self.fp = open( self.filename, 'rb' )
774 774 self.fp.seek( currentPointer )
775 775
776 776 self.fileSize = os.path.getsize( self.filename )
777 777 currentSize = self.fileSize - currentPointer
778 778
779 779 if ( currentSize >= neededSize ):
780 780 self.basicHeaderObj.read(self.fp)
781 781 return 1
782 782
783 783 if self.fileSize == self.fileSizeByHeader:
784 784 # self.flagEoF = True
785 785 return 0
786 786
787 787 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
788 788 sleep( self.delay )
789 789
790 790
791 791 return 0
792 792
793 793 def waitDataBlock(self,pointer_location):
794 794
795 795 currentPointer = pointer_location
796 796
797 797 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
798 798
799 799 for nTries in range( self.nTries ):
800 800 self.fp.close()
801 801 self.fp = open( self.filename, 'rb' )
802 802 self.fp.seek( currentPointer )
803 803
804 804 self.fileSize = os.path.getsize( self.filename )
805 805 currentSize = self.fileSize - currentPointer
806 806
807 807 if ( currentSize >= neededSize ):
808 808 return 1
809 809
810 810 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
811 811 sleep( self.delay )
812 812
813 813 return 0
814 814
815 815 def __jumpToLastBlock(self):
816 816
817 817 if not(self.__isFirstTimeOnline):
818 818 return
819 819
820 820 csize = self.fileSize - self.fp.tell()
821 821 blocksize = self.processingHeaderObj.blockSize
822 822
823 823 #salta el primer bloque de datos
824 824 if csize > self.processingHeaderObj.blockSize:
825 825 self.fp.seek(self.fp.tell() + blocksize)
826 826 else:
827 827 return
828 828
829 829 csize = self.fileSize - self.fp.tell()
830 830 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
831 831 while True:
832 832
833 833 if self.fp.tell()<self.fileSize:
834 834 self.fp.seek(self.fp.tell() + neededsize)
835 835 else:
836 836 self.fp.seek(self.fp.tell() - neededsize)
837 837 break
838 838
839 839 # csize = self.fileSize - self.fp.tell()
840 840 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
841 841 # factor = int(csize/neededsize)
842 842 # if factor > 0:
843 843 # self.fp.seek(self.fp.tell() + factor*neededsize)
844 844
845 845 self.flagIsNewFile = 0
846 846 self.__isFirstTimeOnline = 0
847 847
848 848 def __setNewBlock(self):
849 849
850 850 if self.fp == None:
851 851 return 0
852 852
853 853 if self.online:
854 854 self.__jumpToLastBlock()
855 855
856 856 if self.flagIsNewFile:
857 857 return 1
858 858
859 859 self.lastUTTime = self.basicHeaderObj.utc
860 860 currentSize = self.fileSize - self.fp.tell()
861 861 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
862 862
863 863 if (currentSize >= neededSize):
864 864 self.basicHeaderObj.read(self.fp)
865 865 return 1
866 866
867 867 if self.__waitNewBlock():
868 868 return 1
869 869
870 870 if not(self.setNextFile()):
871 871 return 0
872 872
873 873 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
874 874
875 875 self.flagDiscontinuousBlock = 0
876 876
877 877 if deltaTime > self.maxTimeStep:
878 878 self.flagDiscontinuousBlock = 1
879 879
880 880 return 1
881 881
882 882 def readNextBlock(self):
883 883
884 884 if not(self.__setNewBlock()):
885 885 return 0
886 886
887 887 if not(self.readBlock()):
888 888 return 0
889 889
890 print "[Reading] Block No. %d/%d -> %s" %(self.basicHeaderObj.dataBlock+1,
890 self.getBasicHeader()
891
892 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
891 893 self.processingHeaderObj.dataBlocksPerFile,
892 894 self.dataOut.datatime.ctime())
893 895 return 1
894 896
895 897 def __readFirstHeader(self):
896 898
897 899 self.basicHeaderObj.read(self.fp)
898 900 self.systemHeaderObj.read(self.fp)
899 901 self.radarControllerHeaderObj.read(self.fp)
900 902 self.processingHeaderObj.read(self.fp)
901 903
902 904 self.firstHeaderSize = self.basicHeaderObj.size
903 905
904 906 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
905 907 if datatype == 0:
906 908 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
907 909 elif datatype == 1:
908 910 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
909 911 elif datatype == 2:
910 912 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
911 913 elif datatype == 3:
912 914 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
913 915 elif datatype == 4:
914 916 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
915 917 elif datatype == 5:
916 918 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
917 919 else:
918 920 raise ValueError, 'Data type was not defined'
919 921
920 922 self.dtype = datatype_str
921 923 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
922 924 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
923 925 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
924 926 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
925 927 self.getBlockDimension()
926 928
927 929 def __verifyFile(self, filename, msgFlag=True):
928 930 msg = None
929 931 try:
930 932 fp = open(filename, 'rb')
931 933 currentPosition = fp.tell()
932 934 except IOError:
933 935 traceback.print_exc()
934 936 if msgFlag:
935 937 print "[Reading] The file %s can't be opened" % (filename)
936 938 return False
937 939
938 940 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
939 941
940 942 if neededSize == 0:
941 943 basicHeaderObj = BasicHeader(LOCALTIME)
942 944 systemHeaderObj = SystemHeader()
943 945 radarControllerHeaderObj = RadarControllerHeader()
944 946 processingHeaderObj = ProcessingHeader()
945 947
946 948 try:
947 949 if not( basicHeaderObj.read(fp) ): raise IOError
948 950 if not( systemHeaderObj.read(fp) ): raise IOError
949 951 if not( radarControllerHeaderObj.read(fp) ): raise IOError
950 952 if not( processingHeaderObj.read(fp) ): raise IOError
951 953 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
952 954
953 955 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
954 956
955 957 except IOError:
956 958 traceback.print_exc()
957 959 sys.exit(0)
958 960
959 961 if msgFlag:
960 962 print "[Reading] The file %s is empty or it hasn't enough data" % filename
961 963
962 964 fp.close()
963 965 return False
964 966 else:
965 967 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
966 968
967 969 fp.close()
968 970 fileSize = os.path.getsize(filename)
969 971 currentSize = fileSize - currentPosition
970 972 if currentSize < neededSize:
971 973 if msgFlag and (msg != None):
972 974 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
973 975 return False
974 976
975 977 return True
976 978
977 979 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
978 980
979 981 dateList = []
980 982 pathList = []
981 983
982 984 if not walk:
983 985 #pathList.append(path)
984 986 multi_path = path.split(',')
985 987 for single_path in multi_path:
986 988
987 989 if not os.path.isdir(single_path):
988 990 continue
989 991
990 992 ok = False
991 993 fileList = glob.glob1(single_path, "*"+ext)
992 994
993 995 for thisFile in fileList:
994 996
995 997 if not os.path.isfile(os.path.join(single_path, thisFile)):
996 998 continue
997 999
998 1000 if not isRadarFile(thisFile):
999 1001 continue
1000 1002
1001 1003 ok = True
1002 1004 thisDate = getDateFromRadarFile(thisFile)
1003 1005
1004 1006 if thisDate not in dateList:
1005 1007 dateList.append(thisDate)
1006 1008
1007 1009 if ok:
1008 1010 pathList.append(single_path)
1009 1011
1010 1012 return dateList
1011 1013
1012 1014 multi_path = path.split(',')
1013 1015 for single_path in multi_path:
1014 1016
1015 1017 if not os.path.isdir(single_path):
1016 1018 continue
1017 1019
1018 1020 dirList = []
1019 1021
1020 1022 for thisPath in os.listdir(single_path):
1021 1023
1022 1024 if not os.path.isdir(os.path.join(single_path,thisPath)):
1023 1025 continue
1024 1026
1025 1027 if not isRadarFolder(thisPath):
1026 1028 continue
1027 1029
1028 1030 dirList.append(thisPath)
1029 1031
1030 1032 if not dirList:
1031 1033 return dateList
1032 1034
1033 1035 if startDate and endDate:
1034 1036 thisDate = startDate
1035 1037
1036 1038 while(thisDate <= endDate):
1037 1039 year = thisDate.timetuple().tm_year
1038 1040 doy = thisDate.timetuple().tm_yday
1039 1041
1040 1042 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1041 1043 if len(matchlist) == 0:
1042 1044 thisDate += datetime.timedelta(1)
1043 1045 continue
1044 1046
1045 1047 for match in matchlist:
1046 pathList.append(os.path.join(single_path,match,expLabel))
1048
1049 datapath = os.path.join(single_path, match, expLabel)
1050 fileList = glob.glob1(datapath, "*"+ext)
1051
1052 if len(fileList) < 1:
1053 continue
1054
1055 pathList.append(datapath)
1047 1056 dateList.append(thisDate)
1048 1057
1049 1058 thisDate += datetime.timedelta(1)
1050 1059 else:
1051 1060 for thisDir in dirList:
1061
1062 datapath = os.path.join(single_path, thisDir, expLabel)
1063 fileList = glob.glob1(datapath, "*"+ext)
1064
1065 if len(fileList) < 1:
1066 continue
1067
1052 1068 year = int(thisDir[1:5])
1053 1069 doy = int(thisDir[5:8])
1054 1070 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1055 1071
1056 pathList.append(os.path.join(single_path,thisDir,expLabel))
1072 pathList.append(datapath)
1057 1073 dateList.append(thisDate)
1058 1074
1059 1075 return dateList
1060 1076
1061 1077
1062 1078 def setup(self,
1063 1079 path=None,
1064 1080 startDate=None,
1065 1081 endDate=None,
1066 1082 startTime=datetime.time(0,0,0),
1067 1083 endTime=datetime.time(23,59,59),
1068 1084 set=None,
1069 1085 expLabel = "",
1070 1086 ext = None,
1071 1087 online = False,
1072 1088 delay = 60,
1073 1089 walk = True,
1074 1090 getblock = False,
1075 1091 nTxs = 1):
1076 1092
1077 1093 if path == None:
1078 1094 raise ValueError, "[Reading] The path is not valid"
1079 1095
1080 1096 if ext == None:
1081 1097 ext = self.ext
1082 1098
1083 1099 if online:
1084 1100 print "[Reading] Searching files in online mode..."
1085 1101
1086 1102 for nTries in range( self.nTries ):
1087 1103 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1088 1104
1089 1105 if fullpath:
1090 1106 break
1091 1107
1092 1108 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1093 1109 sleep( self.delay )
1094 1110
1095 1111 if not(fullpath):
1096 1112 print "[Reading] There 'isn't any valid file in %s" % path
1097 1113 return None
1098 1114
1099 1115 self.year = year
1100 1116 self.doy = doy
1101 1117 self.set = set - 1
1102 1118 self.path = path
1103 1119 self.foldercounter = foldercounter
1104 1120 last_set = None
1105 1121
1106 1122 else:
1107 1123 print "[Reading] Searching files in offline mode ..."
1108 1124 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1109 1125 startTime=startTime, endTime=endTime,
1110 1126 set=set, expLabel=expLabel, ext=ext,
1111 1127 walk=walk)
1112 1128
1113 1129 if not(pathList):
1114 1130 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1115 1131 datetime.datetime.combine(startDate,startTime).ctime(),
1116 1132 datetime.datetime.combine(endDate,endTime).ctime())
1117 1133
1118 1134 sys.exit(-1)
1119 1135
1120 1136
1121 1137 self.fileIndex = -1
1122 1138 self.pathList = pathList
1123 1139 self.filenameList = filenameList
1124 1140 file_name = os.path.basename(filenameList[-1])
1125 1141 basename, ext = os.path.splitext(file_name)
1126 1142 last_set = int(basename[-3:])
1127 1143
1128 1144 self.online = online
1129 1145 self.delay = delay
1130 1146 ext = ext.lower()
1131 1147 self.ext = ext
1132 1148 self.getByBlock = getblock
1133 1149 self.nTxs = int(nTxs)
1134 1150
1135 1151 if not(self.setNextFile()):
1136 1152 if (startDate!=None) and (endDate!=None):
1137 1153 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1138 1154 elif startDate != None:
1139 1155 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1140 1156 else:
1141 1157 print "[Reading] No files"
1142 1158
1143 1159 sys.exit(-1)
1144 1160
1145 self.getBasicHeader()
1161 # self.getBasicHeader()
1146 1162
1147 1163 if last_set != None:
1148 1164 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1149 1165 return
1150 1166
1151 1167 def getBasicHeader(self):
1152 1168
1153 1169 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1154 1170
1155 1171 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1156 1172
1157 1173 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1158 1174
1159 1175 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1160 1176
1161 1177 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1162 1178
1163 1179 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1164 1180
1165 1181 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1166 1182
1167 1183 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1168 1184
1169 1185
1170 1186 def getFirstHeader(self):
1171 1187
1172 1188 raise ValueError, "This method has not been implemented"
1173 1189
1174 1190 def getData(self):
1175 1191
1176 1192 raise ValueError, "This method has not been implemented"
1177 1193
1178 1194 def hasNotDataInBuffer(self):
1179 1195
1180 1196 raise ValueError, "This method has not been implemented"
1181 1197
1182 1198 def readBlock(self):
1183 1199
1184 1200 raise ValueError, "This method has not been implemented"
1185 1201
1186 1202 def isEndProcess(self):
1187 1203
1188 1204 return self.flagNoMoreFiles
1189 1205
1190 1206 def printReadBlocks(self):
1191 1207
1192 1208 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1193 1209
1194 1210 def printTotalBlocks(self):
1195 1211
1196 1212 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1197 1213
1198 1214 def printNumberOfBlock(self):
1199 1215
1200 1216 if self.flagIsNewBlock:
1201 print "[Reading] Block No. %d/%d -> %s" %(self.basicHeaderObj.dataBlock+1,
1217 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1202 1218 self.processingHeaderObj.dataBlocksPerFile,
1203 1219 self.dataOut.datatime.ctime())
1204
1205 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1206 1220
1207 1221 def printInfo(self):
1208 1222
1209 1223 if self.__printInfo == False:
1210 1224 return
1211 1225
1212 1226 self.basicHeaderObj.printInfo()
1213 1227 self.systemHeaderObj.printInfo()
1214 1228 self.radarControllerHeaderObj.printInfo()
1215 1229 self.processingHeaderObj.printInfo()
1216 1230
1217 1231 self.__printInfo = False
1218 1232
1219 1233
1220 1234 def run(self, **kwargs):
1221 1235
1222 1236 if not(self.isConfig):
1223 1237
1224 1238 # self.dataOut = dataOut
1225 1239 self.setup(**kwargs)
1226 1240 self.isConfig = True
1227 1241
1228 1242 self.getData()
1229 1243
1230 1244 class JRODataWriter(JRODataIO):
1231 1245
1232 1246 """
1233 1247 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1234 1248 de los datos siempre se realiza por bloques.
1235 1249 """
1236 1250
1237 1251 blockIndex = 0
1238 1252
1239 1253 path = None
1240 1254
1241 1255 setFile = None
1242 1256
1243 1257 profilesPerBlock = None
1244 1258
1245 1259 blocksPerFile = None
1246 1260
1247 1261 nWriteBlocks = 0
1248 1262
1263 fileDate = None
1264
1249 1265 def __init__(self, dataOut=None):
1250 1266 raise ValueError, "Not implemented"
1251 1267
1252 1268
1253 1269 def hasAllDataInBuffer(self):
1254 1270 raise ValueError, "Not implemented"
1255 1271
1256 1272
1257 1273 def setBlockDimension(self):
1258 1274 raise ValueError, "Not implemented"
1259 1275
1260 1276
1261 1277 def writeBlock(self):
1262 1278 raise ValueError, "No implemented"
1263 1279
1264 1280
1265 1281 def putData(self):
1266 1282 raise ValueError, "No implemented"
1267 1283
1268 1284
1269 1285 def getProcessFlags(self):
1270 1286
1271 1287 processFlags = 0
1272 1288
1273 1289 dtype_index = get_dtype_index(self.dtype)
1274 1290 procflag_dtype = get_procflag_dtype(dtype_index)
1275 1291
1276 1292 processFlags += procflag_dtype
1277 1293
1278 1294 if self.dataOut.flagDecodeData:
1279 1295 processFlags += PROCFLAG.DECODE_DATA
1280 1296
1281 1297 if self.dataOut.flagDeflipData:
1282 1298 processFlags += PROCFLAG.DEFLIP_DATA
1283 1299
1284 1300 if self.dataOut.code is not None:
1285 1301 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1286 1302
1287 1303 if self.dataOut.nCohInt > 1:
1288 1304 processFlags += PROCFLAG.COHERENT_INTEGRATION
1289 1305
1290 1306 if self.dataOut.type == "Spectra":
1291 1307 if self.dataOut.nIncohInt > 1:
1292 1308 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1293 1309
1294 1310 if self.dataOut.data_dc is not None:
1295 1311 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1296 1312
1297 1313 if self.dataOut.flagShiftFFT:
1298 1314 processFlags += PROCFLAG.SHIFT_FFT_DATA
1299 1315
1300 1316 return processFlags
1301 1317
1302 1318 def setBasicHeader(self):
1303 1319
1304 1320 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1305 1321 self.basicHeaderObj.version = self.versionFile
1306 self.basicHeaderObj.dataBlock = self.blockIndex
1322 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1307 1323
1308 1324 utc = numpy.floor(self.dataOut.utctime)
1309 1325 milisecond = (self.dataOut.utctime - utc)* 1000.0
1310 1326
1311 1327 self.basicHeaderObj.utc = utc
1312 1328 self.basicHeaderObj.miliSecond = milisecond
1313 1329 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1314 1330 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1315 1331 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1316 1332
1317 1333 def setFirstHeader(self):
1318 1334 """
1319 1335 Obtiene una copia del First Header
1320 1336
1321 1337 Affected:
1322 1338
1323 1339 self.basicHeaderObj
1324 1340 self.systemHeaderObj
1325 1341 self.radarControllerHeaderObj
1326 1342 self.processingHeaderObj self.
1327 1343
1328 1344 Return:
1329 1345 None
1330 1346 """
1331 1347
1332 1348 raise ValueError, "No implemented"
1333 1349
1334 1350 def __writeFirstHeader(self):
1335 1351 """
1336 1352 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1337 1353
1338 1354 Affected:
1339 1355 __dataType
1340 1356
1341 1357 Return:
1342 1358 None
1343 1359 """
1344 1360
1345 1361 # CALCULAR PARAMETROS
1346 1362
1347 1363 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1348 1364 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1349 1365
1350 1366 self.basicHeaderObj.write(self.fp)
1351 1367 self.systemHeaderObj.write(self.fp)
1352 1368 self.radarControllerHeaderObj.write(self.fp)
1353 1369 self.processingHeaderObj.write(self.fp)
1354 1370
1355 1371 def __setNewBlock(self):
1356 1372 """
1357 1373 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1358 1374
1359 1375 Return:
1360 1376 0 : si no pudo escribir nada
1361 1377 1 : Si escribio el Basic el First Header
1362 1378 """
1363 1379 if self.fp == None:
1364 1380 self.setNextFile()
1365 1381
1366 1382 if self.flagIsNewFile:
1367 1383 return 1
1368 1384
1369 1385 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1370 1386 self.basicHeaderObj.write(self.fp)
1371 1387 return 1
1372 1388
1373 1389 if not( self.setNextFile() ):
1374 1390 return 0
1375 1391
1376 1392 return 1
1377 1393
1378 1394
1379 1395 def writeNextBlock(self):
1380 1396 """
1381 1397 Selecciona el bloque siguiente de datos y los escribe en un file
1382 1398
1383 1399 Return:
1384 1400 0 : Si no hizo pudo escribir el bloque de datos
1385 1401 1 : Si no pudo escribir el bloque de datos
1386 1402 """
1387 1403 if not( self.__setNewBlock() ):
1388 1404 return 0
1389 1405
1390 1406 self.writeBlock()
1391 1407
1392 print "[Writing] Block No. %d/%d" %(self.blockIndex, self.processingHeaderObj.dataBlocksPerFile)
1408 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1409 self.processingHeaderObj.dataBlocksPerFile)
1393 1410
1394 1411 return 1
1395 1412
1396 1413 def setNextFile(self):
1397 1414 """
1398 1415 Determina el siguiente file que sera escrito
1399 1416
1400 1417 Affected:
1401 1418 self.filename
1402 1419 self.subfolder
1403 1420 self.fp
1404 1421 self.setFile
1405 1422 self.flagIsNewFile
1406 1423
1407 1424 Return:
1408 1425 0 : Si el archivo no puede ser escrito
1409 1426 1 : Si el archivo esta listo para ser escrito
1410 1427 """
1411 1428 ext = self.ext
1412 1429 path = self.path
1413 1430
1414 1431 if self.fp != None:
1415 1432 self.fp.close()
1416 1433
1417 1434 timeTuple = time.localtime( self.dataOut.utctime)
1418 1435 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1419 1436
1420 1437 fullpath = os.path.join( path, subfolder )
1438 setFile = self.setFile
1439
1421 1440 if not( os.path.exists(fullpath) ):
1422 1441 os.mkdir(fullpath)
1423 self.setFile = -1 #inicializo mi contador de seteo
1442 setFile = -1 #inicializo mi contador de seteo
1424 1443 else:
1425 1444 filesList = os.listdir( fullpath )
1426 1445 if len( filesList ) > 0:
1427 1446 filesList = sorted( filesList, key=str.lower )
1428 1447 filen = filesList[-1]
1429 1448 # el filename debera tener el siguiente formato
1430 1449 # 0 1234 567 89A BCDE (hex)
1431 1450 # x YYYY DDD SSS .ext
1432 1451 if isNumber( filen[8:11] ):
1433 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1452 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1434 1453 else:
1435 self.setFile = -1
1454 setFile = -1
1436 1455 else:
1437 self.setFile = -1 #inicializo mi contador de seteo
1438
1439 setFile = self.setFile
1456 setFile = -1 #inicializo mi contador de seteo
1457
1440 1458 setFile += 1
1459
1460 #If this is a new day it resets some values
1461 if self.dataOut.datatime.date() > self.fileDate:
1462 setFile = 0
1463 self.nTotalBlocks = 0
1441 1464
1442 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1443 timeTuple.tm_year,
1444 timeTuple.tm_yday,
1445 setFile,
1446 ext )
1465 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1447 1466
1448 1467 filename = os.path.join( path, subfolder, filen )
1449 1468
1450 1469 fp = open( filename,'wb' )
1451 1470
1452 1471 self.blockIndex = 0
1453 1472
1454 1473 #guardando atributos
1455 1474 self.filename = filename
1456 1475 self.subfolder = subfolder
1457 1476 self.fp = fp
1458 1477 self.setFile = setFile
1459 1478 self.flagIsNewFile = 1
1479 self.fileDate = self.dataOut.datatime.date()
1460 1480
1461 1481 self.setFirstHeader()
1462 1482
1463 1483 print '[Writing] Opening file: %s'%self.filename
1464 1484
1465 1485 self.__writeFirstHeader()
1466 1486
1467 1487 return 1
1468 1488
1469 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None, datatype=4):
1489 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1470 1490 """
1471 1491 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1472 1492
1473 1493 Inputs:
1474 1494 path : directory where data will be saved
1475 1495 profilesPerBlock : number of profiles per block
1476 1496 set : initial file set
1477 1497 datatype : An integer number that defines data type:
1478 1498 0 : int8 (1 byte)
1479 1499 1 : int16 (2 bytes)
1480 1500 2 : int32 (4 bytes)
1481 1501 3 : int64 (8 bytes)
1482 1502 4 : float32 (4 bytes)
1483 1503 5 : double64 (8 bytes)
1484 1504
1485 1505 Return:
1486 1506 0 : Si no realizo un buen seteo
1487 1507 1 : Si realizo un buen seteo
1488 1508 """
1489 1509
1490 1510 if ext == None:
1491 1511 ext = self.ext
1492 1512
1493 ext = ext.lower()
1494
1495 self.ext = ext
1513 self.ext = ext.lower()
1496 1514
1497 1515 self.path = path
1498 1516
1499 self.setFile = set - 1
1517 if set is None:
1518 self.setFile = -1
1519 else:
1520 self.setFile = set - 1
1500 1521
1501 1522 self.blocksPerFile = blocksPerFile
1502 1523
1503 1524 self.profilesPerBlock = profilesPerBlock
1504 1525
1505 1526 self.dataOut = dataOut
1506
1527 self.fileDate = self.dataOut.datatime.date()
1507 1528 #By default
1508 1529 self.dtype = self.dataOut.dtype
1509 1530
1510 1531 if datatype is not None:
1511 1532 self.dtype = get_numpy_dtype(datatype)
1512 1533
1513 1534 if not(self.setNextFile()):
1514 1535 print "[Writing] There isn't a next file"
1515 1536 return 0
1516 1537
1517 1538 self.setBlockDimension()
1518 1539
1519 1540 return 1
1520 1541
1521 1542 def run(self, dataOut, **kwargs):
1522 1543
1523 1544 if not(self.isConfig):
1524 1545
1525 1546 self.setup(dataOut, **kwargs)
1526 1547 self.isConfig = True
1527 1548
1528 1549 self.putData()
1529 1550
General Comments 0
You need to be logged in to leave comments. Login now