##// END OF EJS Templates
schain
RSS
IO_Writer: default datatype was changed to 3 (int32), instead of 2 (int16).
Miguel Valdez -
r626:33eeed5c06a1
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1529 +1,1529
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
23 23
24 24 LOCALTIME = True
25 25
26 26 def isNumber(cad):
27 27 """
28 28 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
29 29
30 30 Excepciones:
31 31 Si un determinado string no puede ser convertido a numero
32 32 Input:
33 33 str, string al cual se le analiza para determinar si convertible a un numero o no
34 34
35 35 Return:
36 36 True : si el string es uno numerico
37 37 False : no es un string numerico
38 38 """
39 39 try:
40 40 float( cad )
41 41 return True
42 42 except:
43 43 return False
44 44
45 45 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
46 46 """
47 47 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
48 48
49 49 Inputs:
50 50 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
51 51
52 52 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
53 53 segundos contados desde 01/01/1970.
54 54 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
55 55 segundos contados desde 01/01/1970.
56 56
57 57 Return:
58 58 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
59 59 fecha especificado, de lo contrario retorna False.
60 60
61 61 Excepciones:
62 62 Si el archivo no existe o no puede ser abierto
63 63 Si la cabecera no puede ser leida.
64 64
65 65 """
66 66 basicHeaderObj = BasicHeader(LOCALTIME)
67 67
68 68 try:
69 69 fp = open(filename,'rb')
70 70 except IOError:
71 71 traceback.print_exc()
72 72 raise IOError, "The file %s can't be opened" %(filename)
73 73
74 74 sts = basicHeaderObj.read(fp)
75 75 fp.close()
76 76
77 77 if not(sts):
78 78 print "Skipping the file %s because it has not a valid header" %(filename)
79 79 return 0
80 80
81 81 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
82 82 return 0
83 83
84 84 return 1
85 85
86 86 def isFileinThisTime(filename, startTime, endTime):
87 87 """
88 88 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
89 89
90 90 Inputs:
91 91 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
92 92
93 93 startTime : tiempo inicial del rango seleccionado en formato datetime.time
94 94
95 95 endTime : tiempo final del rango seleccionado en formato datetime.time
96 96
97 97 Return:
98 98 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
99 99 fecha especificado, de lo contrario retorna False.
100 100
101 101 Excepciones:
102 102 Si el archivo no existe o no puede ser abierto
103 103 Si la cabecera no puede ser leida.
104 104
105 105 """
106 106
107 107
108 108 try:
109 109 fp = open(filename,'rb')
110 110 except IOError:
111 111 traceback.print_exc()
112 112 raise IOError, "The file %s can't be opened" %(filename)
113 113
114 114 basicHeaderObj = BasicHeader(LOCALTIME)
115 115 sts = basicHeaderObj.read(fp)
116 116 fp.close()
117 117
118 118 thisDatetime = basicHeaderObj.datatime
119 119 thisTime = thisDatetime.time()
120 120
121 121 if not(sts):
122 122 print "Skipping the file %s because it has not a valid header" %(filename)
123 123 return None
124 124
125 125 if not ((startTime <= thisTime) and (endTime > thisTime)):
126 126 return None
127 127
128 128 return thisDatetime
129 129
130 130 def getFileFromSet(path, ext, set):
131 131 validFilelist = []
132 132 fileList = os.listdir(path)
133 133
134 134 # 0 1234 567 89A BCDE
135 135 # H YYYY DDD SSS .ext
136 136
137 137 for thisFile in fileList:
138 138 try:
139 139 year = int(thisFile[1:5])
140 140 doy = int(thisFile[5:8])
141 141 except:
142 142 continue
143 143
144 144 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
145 145 continue
146 146
147 147 validFilelist.append(thisFile)
148 148
149 149 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
150 150
151 151 if len(myfile)!= 0:
152 152 return myfile[0]
153 153 else:
154 154 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
155 155 print 'the filename %s does not exist'%filename
156 156 print '...going to the last file: '
157 157
158 158 if validFilelist:
159 159 validFilelist = sorted( validFilelist, key=str.lower )
160 160 return validFilelist[-1]
161 161
162 162 return None
163 163
164 164 def getlastFileFromPath(path, ext):
165 165 """
166 166 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
167 167 al final de la depuracion devuelve el ultimo file de la lista que quedo.
168 168
169 169 Input:
170 170 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
171 171 ext : extension de los files contenidos en una carpeta
172 172
173 173 Return:
174 174 El ultimo file de una determinada carpeta, no se considera el path.
175 175 """
176 176 validFilelist = []
177 177 fileList = os.listdir(path)
178 178
179 179 # 0 1234 567 89A BCDE
180 180 # H YYYY DDD SSS .ext
181 181
182 182 for thisFile in fileList:
183 183
184 184 year = thisFile[1:5]
185 185 if not isNumber(year):
186 186 continue
187 187
188 188 doy = thisFile[5:8]
189 189 if not isNumber(doy):
190 190 continue
191 191
192 192 year = int(year)
193 193 doy = int(doy)
194 194
195 195 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
196 196 continue
197 197
198 198 validFilelist.append(thisFile)
199 199
200 200 if validFilelist:
201 201 validFilelist = sorted( validFilelist, key=str.lower )
202 202 return validFilelist[-1]
203 203
204 204 return None
205 205
206 206 def checkForRealPath(path, foldercounter, year, doy, set, ext):
207 207 """
208 208 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
209 209 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
210 210 el path exacto de un determinado file.
211 211
212 212 Example :
213 213 nombre correcto del file es .../.../D2009307/P2009307367.ext
214 214
215 215 Entonces la funcion prueba con las siguientes combinaciones
216 216 .../.../y2009307367.ext
217 217 .../.../Y2009307367.ext
218 218 .../.../x2009307/y2009307367.ext
219 219 .../.../x2009307/Y2009307367.ext
220 220 .../.../X2009307/y2009307367.ext
221 221 .../.../X2009307/Y2009307367.ext
222 222 siendo para este caso, la ultima combinacion de letras, identica al file buscado
223 223
224 224 Return:
225 225 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
226 226 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
227 227 para el filename
228 228 """
229 229 fullfilename = None
230 230 find_flag = False
231 231 filename = None
232 232
233 233 prefixDirList = [None,'d','D']
234 234 if ext.lower() == ".r": #voltage
235 235 prefixFileList = ['d','D']
236 236 elif ext.lower() == ".pdata": #spectra
237 237 prefixFileList = ['p','P']
238 238 else:
239 239 return None, filename
240 240
241 241 #barrido por las combinaciones posibles
242 242 for prefixDir in prefixDirList:
243 243 thispath = path
244 244 if prefixDir != None:
245 245 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
246 246 if foldercounter == 0:
247 247 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
248 248 else:
249 249 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
250 250 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
251 251 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
252 252 fullfilename = os.path.join( thispath, filename ) #formo el path completo
253 253
254 254 if os.path.exists( fullfilename ): #verifico que exista
255 255 find_flag = True
256 256 break
257 257 if find_flag:
258 258 break
259 259
260 260 if not(find_flag):
261 261 return None, filename
262 262
263 263 return fullfilename, filename
264 264
265 265 def isRadarFolder(folder):
266 266 try:
267 267 year = int(folder[1:5])
268 268 doy = int(folder[5:8])
269 269 except:
270 270 return 0
271 271
272 272 return 1
273 273
274 274 def isRadarFile(file):
275 275 try:
276 276 year = int(file[1:5])
277 277 doy = int(file[5:8])
278 278 set = int(file[8:11])
279 279 except:
280 280 return 0
281 281
282 282 return 1
283 283
284 284 def getDateFromRadarFile(file):
285 285 try:
286 286 year = int(file[1:5])
287 287 doy = int(file[5:8])
288 288 set = int(file[8:11])
289 289 except:
290 290 return None
291 291
292 292 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
293 293 return thisDate
294 294
295 295 class JRODataIO:
296 296
297 297 c = 3E8
298 298
299 299 isConfig = False
300 300
301 301 basicHeaderObj = None
302 302
303 303 systemHeaderObj = None
304 304
305 305 radarControllerHeaderObj = None
306 306
307 307 processingHeaderObj = None
308 308
309 309 online = 0
310 310
311 311 dtype = None
312 312
313 313 pathList = []
314 314
315 315 filenameList = []
316 316
317 317 filename = None
318 318
319 319 ext = None
320 320
321 321 flagIsNewFile = 1
322 322
323 323 flagDiscontinuousBlock = 0
324 324
325 325 flagIsNewBlock = 0
326 326
327 327 fp = None
328 328
329 329 firstHeaderSize = 0
330 330
331 331 basicHeaderSize = 24
332 332
333 333 versionFile = 1103
334 334
335 335 fileSize = None
336 336
337 337 # ippSeconds = None
338 338
339 339 fileSizeByHeader = None
340 340
341 341 fileIndex = None
342 342
343 343 profileIndex = None
344 344
345 345 blockIndex = None
346 346
347 347 nTotalBlocks = None
348 348
349 349 maxTimeStep = 30
350 350
351 351 lastUTTime = None
352 352
353 353 datablock = None
354 354
355 355 dataOut = None
356 356
357 357 blocksize = None
358 358
359 359 getByBlock = False
360 360
361 361 def __init__(self):
362 362
363 363 raise ValueError, "Not implemented"
364 364
365 365 def run(self):
366 366
367 367 raise ValueError, "Not implemented"
368 368
369 369 def getDtypeWidth(self):
370 370
371 371 dtype_index = get_dtype_index(self.dtype)
372 372 dtype_width = get_dtype_width(dtype_index)
373 373
374 374 return dtype_width
375 375
376 376 class JRODataReader(JRODataIO):
377 377
378 378 nReadBlocks = 0
379 379
380 380 delay = 10 #number of seconds waiting a new file
381 381
382 382 nTries = 3 #quantity tries
383 383
384 384 nFiles = 3 #number of files for searching
385 385
386 386 path = None
387 387
388 388 foldercounter = 0
389 389
390 390 flagNoMoreFiles = 0
391 391
392 392 datetimeList = []
393 393
394 394 __isFirstTimeOnline = 1
395 395
396 396 __printInfo = True
397 397
398 398 profileIndex = None
399 399
400 400 nTxs = 1
401 401
402 402 txIndex = None
403 403
404 404 def __init__(self):
405 405
406 406 """
407 407
408 408 """
409 409
410 410 # raise NotImplementedError, "This method has not been implemented"
411 411
412 412
413 413 def createObjByDefault(self):
414 414 """
415 415
416 416 """
417 417 raise NotImplementedError, "This method has not been implemented"
418 418
419 419 def getBlockDimension(self):
420 420
421 421 raise NotImplementedError, "No implemented"
422 422
423 423 def __searchFilesOffLine(self,
424 424 path,
425 425 startDate=None,
426 426 endDate=None,
427 427 startTime=datetime.time(0,0,0),
428 428 endTime=datetime.time(23,59,59),
429 429 set=None,
430 430 expLabel='',
431 431 ext='.r',
432 432 walk=True):
433 433
434 434 self.filenameList = []
435 435 self.datetimeList = []
436 436
437 437 pathList = []
438 438
439 439 if not walk:
440 440 #pathList.append(path)
441 441 multi_path = path.split(',')
442 442 for single_path in multi_path:
443 443
444 444 if not os.path.isdir(single_path):
445 445 continue
446 446
447 447 pathList.append(single_path)
448 448
449 449 else:
450 450 #dirList = []
451 451 multi_path = path.split(',')
452 452 for single_path in multi_path:
453 453
454 454 if not os.path.isdir(single_path):
455 455 continue
456 456
457 457 dirList = []
458 458 for thisPath in os.listdir(single_path):
459 459 if not os.path.isdir(os.path.join(single_path,thisPath)):
460 460 continue
461 461 if not isRadarFolder(thisPath):
462 462 continue
463 463
464 464 dirList.append(thisPath)
465 465
466 466 if not(dirList):
467 467 return None, None
468 468
469 469 if startDate and endDate:
470 470 thisDate = startDate
471 471
472 472 while(thisDate <= endDate):
473 473 year = thisDate.timetuple().tm_year
474 474 doy = thisDate.timetuple().tm_yday
475 475
476 476 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
477 477 if len(matchlist) == 0:
478 478 thisDate += datetime.timedelta(1)
479 479 continue
480 480 for match in matchlist:
481 481 pathList.append(os.path.join(single_path,match,expLabel))
482 482
483 483 thisDate += datetime.timedelta(1)
484 484 else:
485 485 for thiDir in dirList:
486 486 pathList.append(os.path.join(single_path,thiDir,expLabel))
487 487
488 488 if pathList == []:
489 489 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
490 490 return None, None
491 491
492 492 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
493 493
494 494 filenameList = []
495 495 datetimeList = []
496 496 pathDict = {}
497 497 filenameList_to_sort = []
498 498
499 499 for i in range(len(pathList)):
500 500
501 501 thisPath = pathList[i]
502 502
503 503 fileList = glob.glob1(thisPath, "*%s" %ext)
504 504 if len(fileList) < 1:
505 505 continue
506 506 fileList.sort()
507 507 pathDict.setdefault(fileList[0])
508 508 pathDict[fileList[0]] = i
509 509 filenameList_to_sort.append(fileList[0])
510 510
511 511 filenameList_to_sort.sort()
512 512
513 513 for file in filenameList_to_sort:
514 514 thisPath = pathList[pathDict[file]]
515 515
516 516 fileList = glob.glob1(thisPath, "*%s" %ext)
517 517 fileList.sort()
518 518
519 519 for file in fileList:
520 520
521 521 filename = os.path.join(thisPath,file)
522 522 thisDatetime = isFileinThisTime(filename, startTime, endTime)
523 523
524 524 if not(thisDatetime):
525 525 continue
526 526
527 527 filenameList.append(filename)
528 528 datetimeList.append(thisDatetime)
529 529
530 530 if not(filenameList):
531 531 print "Any file was found for the time range %s - %s" %(startTime, endTime)
532 532 return None, None
533 533
534 534 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
535 535 print
536 536
537 537 for i in range(len(filenameList)):
538 538 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
539 539
540 540 self.filenameList = filenameList
541 541 self.datetimeList = datetimeList
542 542
543 543 return pathList, filenameList
544 544
545 545 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
546 546
547 547 """
548 548 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
549 549 devuelve el archivo encontrado ademas de otros datos.
550 550
551 551 Input:
552 552 path : carpeta donde estan contenidos los files que contiene data
553 553
554 554 expLabel : Nombre del subexperimento (subfolder)
555 555
556 556 ext : extension de los files
557 557
558 558 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
559 559
560 560 Return:
561 561 directory : eL directorio donde esta el file encontrado
562 562 filename : el ultimo file de una determinada carpeta
563 563 year : el anho
564 564 doy : el numero de dia del anho
565 565 set : el set del archivo
566 566
567 567
568 568 """
569 569 dirList = []
570 570
571 571 if not walk:
572 572 fullpath = path
573 573 foldercounter = 0
574 574 else:
575 575 #Filtra solo los directorios
576 576 for thisPath in os.listdir(path):
577 577 if not os.path.isdir(os.path.join(path,thisPath)):
578 578 continue
579 579 if not isRadarFolder(thisPath):
580 580 continue
581 581
582 582 dirList.append(thisPath)
583 583
584 584 if not(dirList):
585 585 return None, None, None, None, None, None
586 586
587 587 dirList = sorted( dirList, key=str.lower )
588 588
589 589 doypath = dirList[-1]
590 590 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
591 591 fullpath = os.path.join(path, doypath, expLabel)
592 592
593 593
594 594 print "%s folder was found: " %(fullpath )
595 595
596 596 if set == None:
597 597 filename = getlastFileFromPath(fullpath, ext)
598 598 else:
599 599 filename = getFileFromSet(fullpath, ext, set)
600 600
601 601 if not(filename):
602 602 return None, None, None, None, None, None
603 603
604 604 print "%s file was found" %(filename)
605 605
606 606 if not(self.__verifyFile(os.path.join(fullpath, filename))):
607 607 return None, None, None, None, None, None
608 608
609 609 year = int( filename[1:5] )
610 610 doy = int( filename[5:8] )
611 611 set = int( filename[8:11] )
612 612
613 613 return fullpath, foldercounter, filename, year, doy, set
614 614
615 615 def __setNextFileOffline(self):
616 616
617 617 idFile = self.fileIndex
618 618
619 619 while (True):
620 620 idFile += 1
621 621 if not(idFile < len(self.filenameList)):
622 622 self.flagNoMoreFiles = 1
623 623 # print "[Reading] No more Files"
624 624 return 0
625 625
626 626 filename = self.filenameList[idFile]
627 627
628 628 if not(self.__verifyFile(filename)):
629 629 continue
630 630
631 631 fileSize = os.path.getsize(filename)
632 632 fp = open(filename,'rb')
633 633 break
634 634
635 635 self.flagIsNewFile = 1
636 636 self.fileIndex = idFile
637 637 self.filename = filename
638 638 self.fileSize = fileSize
639 639 self.fp = fp
640 640
641 641 # print "[Reading] Setting the file: %s"%self.filename
642 642
643 643 return 1
644 644
645 645 def __setNextFileOnline(self):
646 646 """
647 647 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
648 648 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
649 649 siguientes.
650 650
651 651 Affected:
652 652 self.flagIsNewFile
653 653 self.filename
654 654 self.fileSize
655 655 self.fp
656 656 self.set
657 657 self.flagNoMoreFiles
658 658
659 659 Return:
660 660 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
661 661 1 : si el file fue abierto con exito y esta listo a ser leido
662 662
663 663 Excepciones:
664 664 Si un determinado file no puede ser abierto
665 665 """
666 666 nFiles = 0
667 667 fileOk_flag = False
668 668 firstTime_flag = True
669 669
670 670 self.set += 1
671 671
672 672 if self.set > 999:
673 673 self.set = 0
674 674 self.foldercounter += 1
675 675
676 676 #busca el 1er file disponible
677 677 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
678 678 if fullfilename:
679 679 if self.__verifyFile(fullfilename, False):
680 680 fileOk_flag = True
681 681
682 682 #si no encuentra un file entonces espera y vuelve a buscar
683 683 if not(fileOk_flag):
684 684 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
685 685
686 686 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
687 687 tries = self.nTries
688 688 else:
689 689 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
690 690
691 691 for nTries in range( tries ):
692 692 if firstTime_flag:
693 693 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
694 694 sleep( self.delay )
695 695 else:
696 696 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
697 697
698 698 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
699 699 if fullfilename:
700 700 if self.__verifyFile(fullfilename):
701 701 fileOk_flag = True
702 702 break
703 703
704 704 if fileOk_flag:
705 705 break
706 706
707 707 firstTime_flag = False
708 708
709 709 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
710 710 self.set += 1
711 711
712 712 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
713 713 self.set = 0
714 714 self.doy += 1
715 715 self.foldercounter = 0
716 716
717 717 if fileOk_flag:
718 718 self.fileSize = os.path.getsize( fullfilename )
719 719 self.filename = fullfilename
720 720 self.flagIsNewFile = 1
721 721 if self.fp != None: self.fp.close()
722 722 self.fp = open(fullfilename, 'rb')
723 723 self.flagNoMoreFiles = 0
724 724 # print '[Reading] Setting the file: %s' % fullfilename
725 725 else:
726 726 self.fileSize = 0
727 727 self.filename = None
728 728 self.flagIsNewFile = 0
729 729 self.fp = None
730 730 self.flagNoMoreFiles = 1
731 731 # print '[Reading] No more files to read'
732 732
733 733 return fileOk_flag
734 734
735 735 def setNextFile(self):
736 736 if self.fp != None:
737 737 self.fp.close()
738 738
739 739 if self.online:
740 740 newFile = self.__setNextFileOnline()
741 741 else:
742 742 newFile = self.__setNextFileOffline()
743 743
744 744 if not(newFile):
745 745 print '[Reading] No more files to read'
746 746 return 0
747 747
748 748 print '[Reading] Setting the file: %s' % self.filename
749 749
750 750 self.__readFirstHeader()
751 751 self.nReadBlocks = 0
752 752 return 1
753 753
754 754 def __waitNewBlock(self):
755 755 """
756 756 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
757 757
758 758 Si el modo de lectura es OffLine siempre retorn 0
759 759 """
760 760 if not self.online:
761 761 return 0
762 762
763 763 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
764 764 return 0
765 765
766 766 currentPointer = self.fp.tell()
767 767
768 768 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
769 769
770 770 for nTries in range( self.nTries ):
771 771
772 772 self.fp.close()
773 773 self.fp = open( self.filename, 'rb' )
774 774 self.fp.seek( currentPointer )
775 775
776 776 self.fileSize = os.path.getsize( self.filename )
777 777 currentSize = self.fileSize - currentPointer
778 778
779 779 if ( currentSize >= neededSize ):
780 780 self.basicHeaderObj.read(self.fp)
781 781 return 1
782 782
783 783 if self.fileSize == self.fileSizeByHeader:
784 784 # self.flagEoF = True
785 785 return 0
786 786
787 787 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
788 788 sleep( self.delay )
789 789
790 790
791 791 return 0
792 792
793 793 def waitDataBlock(self,pointer_location):
794 794
795 795 currentPointer = pointer_location
796 796
797 797 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
798 798
799 799 for nTries in range( self.nTries ):
800 800 self.fp.close()
801 801 self.fp = open( self.filename, 'rb' )
802 802 self.fp.seek( currentPointer )
803 803
804 804 self.fileSize = os.path.getsize( self.filename )
805 805 currentSize = self.fileSize - currentPointer
806 806
807 807 if ( currentSize >= neededSize ):
808 808 return 1
809 809
810 810 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
811 811 sleep( self.delay )
812 812
813 813 return 0
814 814
815 815 def __jumpToLastBlock(self):
816 816
817 817 if not(self.__isFirstTimeOnline):
818 818 return
819 819
820 820 csize = self.fileSize - self.fp.tell()
821 821 blocksize = self.processingHeaderObj.blockSize
822 822
823 823 #salta el primer bloque de datos
824 824 if csize > self.processingHeaderObj.blockSize:
825 825 self.fp.seek(self.fp.tell() + blocksize)
826 826 else:
827 827 return
828 828
829 829 csize = self.fileSize - self.fp.tell()
830 830 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
831 831 while True:
832 832
833 833 if self.fp.tell()<self.fileSize:
834 834 self.fp.seek(self.fp.tell() + neededsize)
835 835 else:
836 836 self.fp.seek(self.fp.tell() - neededsize)
837 837 break
838 838
839 839 # csize = self.fileSize - self.fp.tell()
840 840 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
841 841 # factor = int(csize/neededsize)
842 842 # if factor > 0:
843 843 # self.fp.seek(self.fp.tell() + factor*neededsize)
844 844
845 845 self.flagIsNewFile = 0
846 846 self.__isFirstTimeOnline = 0
847 847
848 848 def __setNewBlock(self):
849 849
850 850 if self.fp == None:
851 851 return 0
852 852
853 853 if self.online:
854 854 self.__jumpToLastBlock()
855 855
856 856 if self.flagIsNewFile:
857 857 return 1
858 858
859 859 self.lastUTTime = self.basicHeaderObj.utc
860 860 currentSize = self.fileSize - self.fp.tell()
861 861 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
862 862
863 863 if (currentSize >= neededSize):
864 864 self.basicHeaderObj.read(self.fp)
865 865 return 1
866 866
867 867 if self.__waitNewBlock():
868 868 return 1
869 869
870 870 if not(self.setNextFile()):
871 871 return 0
872 872
873 873 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
874 874
875 875 self.flagDiscontinuousBlock = 0
876 876
877 877 if deltaTime > self.maxTimeStep:
878 878 self.flagDiscontinuousBlock = 1
879 879
880 880 return 1
881 881
882 882 def readNextBlock(self):
883 883
884 884 if not(self.__setNewBlock()):
885 885 return 0
886 886
887 887 if not(self.readBlock()):
888 888 return 0
889 889
890 890 print "[Reading] Block No. %d/%d -> %s" %(self.basicHeaderObj.dataBlock+1,
891 891 self.processingHeaderObj.dataBlocksPerFile,
892 892 self.dataOut.datatime.ctime())
893 893 return 1
894 894
895 895 def __readFirstHeader(self):
896 896
897 897 self.basicHeaderObj.read(self.fp)
898 898 self.systemHeaderObj.read(self.fp)
899 899 self.radarControllerHeaderObj.read(self.fp)
900 900 self.processingHeaderObj.read(self.fp)
901 901
902 902 self.firstHeaderSize = self.basicHeaderObj.size
903 903
904 904 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
905 905 if datatype == 0:
906 906 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
907 907 elif datatype == 1:
908 908 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
909 909 elif datatype == 2:
910 910 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
911 911 elif datatype == 3:
912 912 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
913 913 elif datatype == 4:
914 914 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
915 915 elif datatype == 5:
916 916 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
917 917 else:
918 918 raise ValueError, 'Data type was not defined'
919 919
920 920 self.dtype = datatype_str
921 921 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
922 922 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
923 923 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
924 924 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
925 925 self.getBlockDimension()
926 926
927 927 def __verifyFile(self, filename, msgFlag=True):
928 928 msg = None
929 929 try:
930 930 fp = open(filename, 'rb')
931 931 currentPosition = fp.tell()
932 932 except IOError:
933 933 traceback.print_exc()
934 934 if msgFlag:
935 935 print "[Reading] The file %s can't be opened" % (filename)
936 936 return False
937 937
938 938 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
939 939
940 940 if neededSize == 0:
941 941 basicHeaderObj = BasicHeader(LOCALTIME)
942 942 systemHeaderObj = SystemHeader()
943 943 radarControllerHeaderObj = RadarControllerHeader()
944 944 processingHeaderObj = ProcessingHeader()
945 945
946 946 try:
947 947 if not( basicHeaderObj.read(fp) ): raise IOError
948 948 if not( systemHeaderObj.read(fp) ): raise IOError
949 949 if not( radarControllerHeaderObj.read(fp) ): raise IOError
950 950 if not( processingHeaderObj.read(fp) ): raise IOError
951 951 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
952 952
953 953 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
954 954
955 955 except IOError:
956 956 traceback.print_exc()
957 957 sys.exit(0)
958 958
959 959 if msgFlag:
960 960 print "[Reading] The file %s is empty or it hasn't enough data" % filename
961 961
962 962 fp.close()
963 963 return False
964 964 else:
965 965 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
966 966
967 967 fp.close()
968 968 fileSize = os.path.getsize(filename)
969 969 currentSize = fileSize - currentPosition
970 970 if currentSize < neededSize:
971 971 if msgFlag and (msg != None):
972 972 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
973 973 return False
974 974
975 975 return True
976 976
977 977 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
978 978
979 979 dateList = []
980 980 pathList = []
981 981
982 982 if not walk:
983 983 #pathList.append(path)
984 984 multi_path = path.split(',')
985 985 for single_path in multi_path:
986 986
987 987 if not os.path.isdir(single_path):
988 988 continue
989 989
990 990 ok = False
991 991 fileList = glob.glob1(single_path, "*"+ext)
992 992
993 993 for thisFile in fileList:
994 994
995 995 if not os.path.isfile(os.path.join(single_path, thisFile)):
996 996 continue
997 997
998 998 if not isRadarFile(thisFile):
999 999 continue
1000 1000
1001 1001 ok = True
1002 1002 thisDate = getDateFromRadarFile(thisFile)
1003 1003
1004 1004 if thisDate not in dateList:
1005 1005 dateList.append(thisDate)
1006 1006
1007 1007 if ok:
1008 1008 pathList.append(single_path)
1009 1009
1010 1010 return dateList
1011 1011
1012 1012 multi_path = path.split(',')
1013 1013 for single_path in multi_path:
1014 1014
1015 1015 if not os.path.isdir(single_path):
1016 1016 continue
1017 1017
1018 1018 dirList = []
1019 1019
1020 1020 for thisPath in os.listdir(single_path):
1021 1021
1022 1022 if not os.path.isdir(os.path.join(single_path,thisPath)):
1023 1023 continue
1024 1024
1025 1025 if not isRadarFolder(thisPath):
1026 1026 continue
1027 1027
1028 1028 dirList.append(thisPath)
1029 1029
1030 1030 if not dirList:
1031 1031 return dateList
1032 1032
1033 1033 if startDate and endDate:
1034 1034 thisDate = startDate
1035 1035
1036 1036 while(thisDate <= endDate):
1037 1037 year = thisDate.timetuple().tm_year
1038 1038 doy = thisDate.timetuple().tm_yday
1039 1039
1040 1040 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1041 1041 if len(matchlist) == 0:
1042 1042 thisDate += datetime.timedelta(1)
1043 1043 continue
1044 1044
1045 1045 for match in matchlist:
1046 1046 pathList.append(os.path.join(single_path,match,expLabel))
1047 1047 dateList.append(thisDate)
1048 1048
1049 1049 thisDate += datetime.timedelta(1)
1050 1050 else:
1051 1051 for thisDir in dirList:
1052 1052 year = int(thisDir[1:5])
1053 1053 doy = int(thisDir[5:8])
1054 1054 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1055 1055
1056 1056 pathList.append(os.path.join(single_path,thisDir,expLabel))
1057 1057 dateList.append(thisDate)
1058 1058
1059 1059 return dateList
1060 1060
1061 1061
1062 1062 def setup(self,
1063 1063 path=None,
1064 1064 startDate=None,
1065 1065 endDate=None,
1066 1066 startTime=datetime.time(0,0,0),
1067 1067 endTime=datetime.time(23,59,59),
1068 1068 set=None,
1069 1069 expLabel = "",
1070 1070 ext = None,
1071 1071 online = False,
1072 1072 delay = 60,
1073 1073 walk = True,
1074 1074 getblock = False,
1075 1075 nTxs = 1):
1076 1076
1077 1077 if path == None:
1078 1078 raise ValueError, "[Reading] The path is not valid"
1079 1079
1080 1080 if ext == None:
1081 1081 ext = self.ext
1082 1082
1083 1083 if online:
1084 1084 print "[Reading] Searching files in online mode..."
1085 1085
1086 1086 for nTries in range( self.nTries ):
1087 1087 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1088 1088
1089 1089 if fullpath:
1090 1090 break
1091 1091
1092 1092 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1093 1093 sleep( self.delay )
1094 1094
1095 1095 if not(fullpath):
1096 1096 print "[Reading] There 'isn't any valid file in %s" % path
1097 1097 return None
1098 1098
1099 1099 self.year = year
1100 1100 self.doy = doy
1101 1101 self.set = set - 1
1102 1102 self.path = path
1103 1103 self.foldercounter = foldercounter
1104 1104 last_set = None
1105 1105
1106 1106 else:
1107 1107 print "[Reading] Searching files in offline mode ..."
1108 1108 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1109 1109 startTime=startTime, endTime=endTime,
1110 1110 set=set, expLabel=expLabel, ext=ext,
1111 1111 walk=walk)
1112 1112
1113 1113 if not(pathList):
1114 1114 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1115 1115 datetime.datetime.combine(startDate,startTime).ctime(),
1116 1116 datetime.datetime.combine(endDate,endTime).ctime())
1117 1117
1118 1118 sys.exit(-1)
1119 1119
1120 1120
1121 1121 self.fileIndex = -1
1122 1122 self.pathList = pathList
1123 1123 self.filenameList = filenameList
1124 1124 file_name = os.path.basename(filenameList[-1])
1125 1125 basename, ext = os.path.splitext(file_name)
1126 1126 last_set = int(basename[-3:])
1127 1127
1128 1128 self.online = online
1129 1129 self.delay = delay
1130 1130 ext = ext.lower()
1131 1131 self.ext = ext
1132 1132 self.getByBlock = getblock
1133 1133 self.nTxs = int(nTxs)
1134 1134
1135 1135 if not(self.setNextFile()):
1136 1136 if (startDate!=None) and (endDate!=None):
1137 1137 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1138 1138 elif startDate != None:
1139 1139 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1140 1140 else:
1141 1141 print "[Reading] No files"
1142 1142
1143 1143 sys.exit(-1)
1144 1144
1145 1145 self.getBasicHeader()
1146 1146
1147 1147 if last_set != None:
1148 1148 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1149 1149 return
1150 1150
1151 1151 def getBasicHeader(self):
1152 1152
1153 1153 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1154 1154
1155 1155 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1156 1156
1157 1157 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1158 1158
1159 1159 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1160 1160
1161 1161 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1162 1162
1163 1163 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1164 1164
1165 1165 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1166 1166
1167 1167 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1168 1168
1169 1169
1170 1170 def getFirstHeader(self):
1171 1171
1172 1172 raise ValueError, "This method has not been implemented"
1173 1173
1174 1174 def getData(self):
1175 1175
1176 1176 raise ValueError, "This method has not been implemented"
1177 1177
1178 1178 def hasNotDataInBuffer(self):
1179 1179
1180 1180 raise ValueError, "This method has not been implemented"
1181 1181
1182 1182 def readBlock(self):
1183 1183
1184 1184 raise ValueError, "This method has not been implemented"
1185 1185
1186 1186 def isEndProcess(self):
1187 1187
1188 1188 return self.flagNoMoreFiles
1189 1189
1190 1190 def printReadBlocks(self):
1191 1191
1192 1192 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1193 1193
1194 1194 def printTotalBlocks(self):
1195 1195
1196 1196 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1197 1197
1198 1198 def printNumberOfBlock(self):
1199 1199
1200 1200 if self.flagIsNewBlock:
1201 1201 print "[Reading] Block No. %d/%d -> %s" %(self.basicHeaderObj.dataBlock+1,
1202 1202 self.processingHeaderObj.dataBlocksPerFile,
1203 1203 self.dataOut.datatime.ctime())
1204 1204
1205 1205 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1206 1206
1207 1207 def printInfo(self):
1208 1208
1209 1209 if self.__printInfo == False:
1210 1210 return
1211 1211
1212 1212 self.basicHeaderObj.printInfo()
1213 1213 self.systemHeaderObj.printInfo()
1214 1214 self.radarControllerHeaderObj.printInfo()
1215 1215 self.processingHeaderObj.printInfo()
1216 1216
1217 1217 self.__printInfo = False
1218 1218
1219 1219
1220 1220 def run(self, **kwargs):
1221 1221
1222 1222 if not(self.isConfig):
1223 1223
1224 1224 # self.dataOut = dataOut
1225 1225 self.setup(**kwargs)
1226 1226 self.isConfig = True
1227 1227
1228 1228 self.getData()
1229 1229
1230 1230 class JRODataWriter(JRODataIO):
1231 1231
1232 1232 """
1233 1233 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1234 1234 de los datos siempre se realiza por bloques.
1235 1235 """
1236 1236
1237 1237 blockIndex = 0
1238 1238
1239 1239 path = None
1240 1240
1241 1241 setFile = None
1242 1242
1243 1243 profilesPerBlock = None
1244 1244
1245 1245 blocksPerFile = None
1246 1246
1247 1247 nWriteBlocks = 0
1248 1248
1249 1249 def __init__(self, dataOut=None):
1250 1250 raise ValueError, "Not implemented"
1251 1251
1252 1252
1253 1253 def hasAllDataInBuffer(self):
1254 1254 raise ValueError, "Not implemented"
1255 1255
1256 1256
1257 1257 def setBlockDimension(self):
1258 1258 raise ValueError, "Not implemented"
1259 1259
1260 1260
1261 1261 def writeBlock(self):
1262 1262 raise ValueError, "No implemented"
1263 1263
1264 1264
1265 1265 def putData(self):
1266 1266 raise ValueError, "No implemented"
1267 1267
1268 1268
1269 1269 def getProcessFlags(self):
1270 1270
1271 1271 processFlags = 0
1272 1272
1273 1273 dtype_index = get_dtype_index(self.dtype)
1274 1274 procflag_dtype = get_procflag_dtype(dtype_index)
1275 1275
1276 1276 processFlags += procflag_dtype
1277 1277
1278 1278 if self.dataOut.flagDecodeData:
1279 1279 processFlags += PROCFLAG.DECODE_DATA
1280 1280
1281 1281 if self.dataOut.flagDeflipData:
1282 1282 processFlags += PROCFLAG.DEFLIP_DATA
1283 1283
1284 1284 if self.dataOut.code is not None:
1285 1285 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1286 1286
1287 1287 if self.dataOut.nCohInt > 1:
1288 1288 processFlags += PROCFLAG.COHERENT_INTEGRATION
1289 1289
1290 1290 if self.dataOut.type == "Spectra":
1291 1291 if self.dataOut.nIncohInt > 1:
1292 1292 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1293 1293
1294 1294 if self.dataOut.data_dc is not None:
1295 1295 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1296 1296
1297 1297 if self.dataOut.flagShiftFFT:
1298 1298 processFlags += PROCFLAG.SHIFT_FFT_DATA
1299 1299
1300 1300 return processFlags
1301 1301
1302 1302 def setBasicHeader(self):
1303 1303
1304 1304 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1305 1305 self.basicHeaderObj.version = self.versionFile
1306 1306 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1307 1307
1308 1308 utc = numpy.floor(self.dataOut.utctime)
1309 1309 milisecond = (self.dataOut.utctime - utc)* 1000.0
1310 1310
1311 1311 self.basicHeaderObj.utc = utc
1312 1312 self.basicHeaderObj.miliSecond = milisecond
1313 1313 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1314 1314 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1315 1315 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1316 1316
1317 1317 def setFirstHeader(self):
1318 1318 """
1319 1319 Obtiene una copia del First Header
1320 1320
1321 1321 Affected:
1322 1322
1323 1323 self.basicHeaderObj
1324 1324 self.systemHeaderObj
1325 1325 self.radarControllerHeaderObj
1326 1326 self.processingHeaderObj self.
1327 1327
1328 1328 Return:
1329 1329 None
1330 1330 """
1331 1331
1332 1332 raise ValueError, "No implemented"
1333 1333
1334 1334 def __writeFirstHeader(self):
1335 1335 """
1336 1336 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1337 1337
1338 1338 Affected:
1339 1339 __dataType
1340 1340
1341 1341 Return:
1342 1342 None
1343 1343 """
1344 1344
1345 1345 # CALCULAR PARAMETROS
1346 1346
1347 1347 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1348 1348 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1349 1349
1350 1350 self.basicHeaderObj.write(self.fp)
1351 1351 self.systemHeaderObj.write(self.fp)
1352 1352 self.radarControllerHeaderObj.write(self.fp)
1353 1353 self.processingHeaderObj.write(self.fp)
1354 1354
1355 1355 def __setNewBlock(self):
1356 1356 """
1357 1357 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1358 1358
1359 1359 Return:
1360 1360 0 : si no pudo escribir nada
1361 1361 1 : Si escribio el Basic el First Header
1362 1362 """
1363 1363 if self.fp == None:
1364 1364 self.setNextFile()
1365 1365
1366 1366 if self.flagIsNewFile:
1367 1367 return 1
1368 1368
1369 1369 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1370 1370 self.basicHeaderObj.write(self.fp)
1371 1371 return 1
1372 1372
1373 1373 if not( self.setNextFile() ):
1374 1374 return 0
1375 1375
1376 1376 return 1
1377 1377
1378 1378
1379 1379 def writeNextBlock(self):
1380 1380 """
1381 1381 Selecciona el bloque siguiente de datos y los escribe en un file
1382 1382
1383 1383 Return:
1384 1384 0 : Si no hizo pudo escribir el bloque de datos
1385 1385 1 : Si no pudo escribir el bloque de datos
1386 1386 """
1387 1387 if not( self.__setNewBlock() ):
1388 1388 return 0
1389 1389
1390 1390 self.writeBlock()
1391 1391
1392 1392 print "[Writing] Block No. %d/%d" %(self.blockIndex, self.processingHeaderObj.dataBlocksPerFile)
1393 1393
1394 1394 return 1
1395 1395
1396 1396 def setNextFile(self):
1397 1397 """
1398 1398 Determina el siguiente file que sera escrito
1399 1399
1400 1400 Affected:
1401 1401 self.filename
1402 1402 self.subfolder
1403 1403 self.fp
1404 1404 self.setFile
1405 1405 self.flagIsNewFile
1406 1406
1407 1407 Return:
1408 1408 0 : Si el archivo no puede ser escrito
1409 1409 1 : Si el archivo esta listo para ser escrito
1410 1410 """
1411 1411 ext = self.ext
1412 1412 path = self.path
1413 1413
1414 1414 if self.fp != None:
1415 1415 self.fp.close()
1416 1416
1417 1417 timeTuple = time.localtime( self.dataOut.utctime)
1418 1418 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1419 1419
1420 1420 fullpath = os.path.join( path, subfolder )
1421 1421 if not( os.path.exists(fullpath) ):
1422 1422 os.mkdir(fullpath)
1423 1423 self.setFile = -1 #inicializo mi contador de seteo
1424 1424 else:
1425 1425 filesList = os.listdir( fullpath )
1426 1426 if len( filesList ) > 0:
1427 1427 filesList = sorted( filesList, key=str.lower )
1428 1428 filen = filesList[-1]
1429 1429 # el filename debera tener el siguiente formato
1430 1430 # 0 1234 567 89A BCDE (hex)
1431 1431 # x YYYY DDD SSS .ext
1432 1432 if isNumber( filen[8:11] ):
1433 1433 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1434 1434 else:
1435 1435 self.setFile = -1
1436 1436 else:
1437 1437 self.setFile = -1 #inicializo mi contador de seteo
1438 1438
1439 1439 setFile = self.setFile
1440 1440 setFile += 1
1441 1441
1442 1442 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1443 1443 timeTuple.tm_year,
1444 1444 timeTuple.tm_yday,
1445 1445 setFile,
1446 1446 ext )
1447 1447
1448 1448 filename = os.path.join( path, subfolder, filen )
1449 1449
1450 1450 fp = open( filename,'wb' )
1451 1451
1452 1452 self.blockIndex = 0
1453 1453
1454 1454 #guardando atributos
1455 1455 self.filename = filename
1456 1456 self.subfolder = subfolder
1457 1457 self.fp = fp
1458 1458 self.setFile = setFile
1459 1459 self.flagIsNewFile = 1
1460 1460
1461 1461 self.setFirstHeader()
1462 1462
1463 1463 print '[Writing] Opening file: %s'%self.filename
1464 1464
1465 1465 self.__writeFirstHeader()
1466 1466
1467 1467 return 1
1468 1468
1469 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None, datatype=2):
1469 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None, datatype=3):
1470 1470 """
1471 1471 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1472 1472
1473 1473 Inputs:
1474 1474 path : directory where data will be saved
1475 1475 profilesPerBlock : number of profiles per block
1476 set : file set
1476 set : initial file set
1477 1477 datatype : An integer number that defines data type:
1478 1478 0 : int8 (1 byte)
1479 1479 1 : int16 (2 bytes)
1480 1480 2 : int32 (4 bytes)
1481 1481 3 : int64 (8 bytes)
1482 4 : float (4 bytes)
1483 5 : double (8 bytes)
1482 4 : float32 (4 bytes)
1483 5 : double64 (8 bytes)
1484 1484
1485 1485 Return:
1486 1486 0 : Si no realizo un buen seteo
1487 1487 1 : Si realizo un buen seteo
1488 1488 """
1489 1489
1490 1490 if ext == None:
1491 1491 ext = self.ext
1492 1492
1493 1493 ext = ext.lower()
1494 1494
1495 1495 self.ext = ext
1496 1496
1497 1497 self.path = path
1498 1498
1499 1499 self.setFile = set - 1
1500 1500
1501 1501 self.blocksPerFile = blocksPerFile
1502 1502
1503 1503 self.profilesPerBlock = profilesPerBlock
1504 1504
1505 1505 self.dataOut = dataOut
1506 1506
1507 1507 #By default
1508 1508 self.dtype = self.dataOut.dtype
1509 1509
1510 1510 if datatype is not None:
1511 1511 self.dtype = get_numpy_dtype(datatype)
1512 1512
1513 1513 if not(self.setNextFile()):
1514 1514 print "[Writing] There isn't a next file"
1515 1515 return 0
1516 1516
1517 1517 self.setBlockDimension()
1518 1518
1519 1519 return 1
1520 1520
1521 1521 def run(self, dataOut, **kwargs):
1522 1522
1523 1523 if not(self.isConfig):
1524 1524
1525 1525 self.setup(dataOut, **kwargs)
1526 1526 self.isConfig = True
1527 1527
1528 1528 self.putData()
1529 1529
General Comments 0
You need to be logged in to leave comments. Login now