##// END OF EJS Templates
schain
RSS
SPECTRAIO.py:...
Victor Sarmiento -
r47:580af48a6d5c
parent child
Show More
Show file before | Show file after | Hide comments
@@ -22,6 +22,39 from DataIO import DataWriter
22 22 from Model.Spectra import Spectra
23 23
24 24
25 def isFileOK(filename):
26 """
27 Determina si la cabecera de un archivo es valido o no, si lo es entonces seria un archivo que podria contener data,
28 si no seria un archivo invalido
29
30 Return:
31 True : si es un archivo valido
32 False : si no es un archivo valido
33
34 Exceptions:
35 Si al leer la cabecera esta no coincide con el tipo de las variables que la contienen entonces se dispara
36 una exception
37 """
38 m_BasicHeader = BasicHeader()
39 m_ProcessingHeader = ProcessingHeader()
40 m_RadarControllerHeader = RadarControllerHeader()
41 m_SystemHeader = SystemHeader()
42 fp = None
43
44 try:
45 fp = open( filename,'rb' ) #lectura binaria
46 m_BasicHeader.read(fp)
47 m_SystemHeader.read(fp)
48 m_RadarControllerHeader.read(fp)
49 m_ProcessingHeader.read(fp)
50 fp.close()
51 except:
52 if fp != None: fp.close()
53 return False
54
55 return True
56
57
25 58 def getlastFileFromPath( pathList, ext ):
26 59 """
27 60 Depura el pathList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
@@ -297,7 +330,7 class SpectraReader( DataReader ):
297 330
298 331 self.datablock_id = 9999
299 332
300 self.__delay = 7 #seconds
333 self.__delay = 2 #seconds
301 334 self.__nTries = 3 #quantity tries
302 335 self.__nFiles = 3 #number of files for searching
303 336 self.__year = 0
@@ -305,7 +338,7 class SpectraReader( DataReader ):
305 338 self.__set = 0
306 339 self.__ext = None
307 340 self.__path = None
308
341 self.nBlocks = 0
309 342
310 343 def __rdSystemHeader( self, fp=None ):
311 344 """
@@ -455,8 +488,6 class SpectraReader( DataReader ):
455 488
456 489 self.__blocksize = self.__pts2read_SelfSpectra + self.__pts2read_CrossSpectra + self.__pts2read_DCchannels
457 490
458 print "SIZEEEE ",self.__blocksize, self.m_ProcessingHeader.blockSize
459
460 491 self.m_Spectra.nChannels = self.nChannels
461 492 self.m_Spectra.nPairs = self.nPairs
462 493
@@ -485,9 +516,10 class SpectraReader( DataReader ):
485 516 countFiles = 0
486 517 countTries = 0
487 518
488 fileStatus = 0
519 #fileStatus = 0
489 520 notFirstTime_flag = False
490 bChangeDir = False
521 fileOk_flag = False
522 changeDir_flag = False
491 523
492 524 fileSize = 0
493 525 fp = None
@@ -507,12 +539,35 class SpectraReader( DataReader ):
507 539 if countFiles > self.__nFiles: #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
508 540 self.__set = 0
509 541 self.__doy += 1
510 bChangeDir = True
542 changeDir_flag = True
511 543
512 544 file = None
513 545 filename = None
546 fileOk_flag = False
547
548 #busca el 1er file disponible
549 file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext )
550
551 if file == None:
552 if notFirstTime_flag: #si no es la primera vez que busca el file entonces no espera y busca for el siguiente file
553 print "\tsearching next \"%s\" file ..." % ( filename )
554 continue
555 else: #si es la primera vez que busca el file entonces espera self.__nTries veces hasta encontrarlo o no
556 for nTries in range( self.__nTries ):
557 print "\twaiting new \"%s\" file, try %03d ..." % ( filename, nTries+1 )
558 time.sleep( self.__delay )
559
560 file, filename = checkForRealPath( self.__path, self.__year, self.__doy, self.__set, self.__ext )
561 if file != None:
562 fileOk_flag = True
563 break
564
565 if not( fileOk_flag ): #no encontro ningun file valido a leer
566 notFirstTime_flag = True
567 continue
514 568
515 countTries = 0
569
570 """countTries = 0
516 571
517 572 #espero hasta encontrar el 1er file disponible
518 573 while( True ):
@@ -536,51 +591,47 class SpectraReader( DataReader ):
536 591 if countTries >= self.__nTries: #se realizaron n intentos y no hubo un file nuevo
537 592 notFirstTime_flag = True
538 593 continue #vuelvo al inico del while principal
594 """
539 595
540 countTries = 0
541
542 #una vez que se obtuvo el 1er file valido se procede a checkear su contenido, y se espera una cierta cantidad
543 #de tiempo por una cierta cantidad de veces hasta que el contenido del file sea un contenido valido
544 while( True ):
545 countTries += 1
546 if countTries > self.__nTries:
547 break
548
549 try:
550 fp = open(file)
551 except:
552 print "The file \"%s\" can't be opened" % file
553 break
554
555 fileSize = os.path.getsize( file )
556 currentSize = fileSize - fp.tell()
596 #una vez que se obtuvo el 1er file valido se procede a checkear si si tamanho es suficiente para empezar a leerlo
597 currentSize = os.path.getsize( file )
557 598 neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
558 599
600 #si el tamanho es suficiente entonces deja de buscar
559 601 if currentSize > neededSize:
560 fileStatus = 1
602 fileOk_flag = True
561 603 break
562 604
563 fp.close()
564
565 if bChangeDir: #si al buscar un file cambie de directorio ya no espero y salgo del bucle while
605 fileOk_flag = False
606 #si el file no tiene el tamanho necesario se espera una cierta cantidad de tiempo
607 #por una cierta cantidad de veces hasta que el contenido del file sea valido
608 if changeDir_flag: #si al buscar un file cambie de directorio ya no espero y sigo con el siguiente file
566 609 print "\tsearching next \"%s\" file ..." % filename
567 break
610 changeDir_flag = False
611 continue
568 612
569 print "\twaiting for block of \"%s\" file ..." % filename
613 for nTries in range( self.__nTries ):
614 print "\twaiting for the First Header block of \"%s\" file, try %03d ..." % ( filename, nTries+1 )
570 615 time.sleep( self.__delay )
571 616
572 if fileStatus == 1:
617 currentSize = os.path.getsize( file )
618 neededSize = self.m_ProcessingHeader.blockSize + self.firstHeaderSize
619
620 if currentSize > neededSize:
621 fileOk_flag = True
622 break
623
624 if fileOk_flag: #si encontro un file valido sale del bucle y deja de buscar
573 625 break
574 626
575 627 print "Skipping the file \"%s\" due to this files is empty" % filename
576 628 countFiles = 0
577 629
578
579 if fileStatus == 1:
580 self.fileSize = fileSize
630 if fileOk_flag:
631 self.fileSize = os.path.getsize( file ) #fileSize
581 632 self.filename = file#name
582 633 self.__flagIsNewFile = 1
583 self.__fp = fp
634 self.__fp = open(file)
584 635 self.noMoreFiles = 0
585 636 print 'Setting the file: %s' % file #name
586 637 else:
@@ -590,7 +641,7 class SpectraReader( DataReader ):
590 641 self.noMoreFiles = 1
591 642 print 'No more Files'
592 643
593 return fileStatus
644 return fileOk_flag
594 645
595 646
596 647 def __setNextFileOffline( self ):
@@ -619,6 +670,7 class SpectraReader( DataReader ):
619 670
620 671 if not( idFile < len(self.filenameList) ):
621 672 self.noMoreFiles = 1
673 print 'No more Files'
622 674 return 0
623 675
624 676 filename = self.filenameList[idFile]
@@ -638,6 +690,7 class SpectraReader( DataReader ):
638 690 continue
639 691
640 692 break
693
641 694 self.__flagIsNewFile = 1
642 695 self.__idFile = idFile
643 696 self.filename = filename
@@ -672,11 +725,14 class SpectraReader( DataReader ):
672 725 else:
673 726 newFile = self.__setNextFileOffline()
674 727
728 if self.noMoreFiles:
729 sys.exit(0)
730
675 731 if not(newFile):
676 732 return 0
677 733
678 734 self.__readFirstHeader()
679
735 self.nBlocks = 0
680 736 return 1
681 737
682 738
@@ -706,15 +762,22 class SpectraReader( DataReader ):
706 762 if ( currentSize >= neededSize ):
707 763 self.__rdBasicHeader()
708 764 return 1
709 elif self.online:
710 nTries = 0
711 while( nTries < self.__nTries ):
712 nTries += 1
713 print "Waiting for the next block, try %03d ..." % nTries
765
766 #si es OnLine y ademas aun no se han leido un bloque completo entonces se espera por uno valido
767 elif (self.nBlocks != self.m_ProcessingHeader.dataBlocksPerFile) and self.online:
768 for nTries in range( self.__nTries ):
769
770 fpointer = self.__fp.tell()
771 self.__fp.close()
772
773 print "\tWaiting for the next block, try %03d ..." % (nTries+1)
714 774 time.sleep( self.__delay )
715 775
716 fileSize = os.path.getsize(self.filename)
717 currentSize = fileSize - self.__fp.tell()
776 self.__fp = open( self.filename, 'rb' )
777 self.__fp.seek( fpointer )
778
779 self.fileSize = os.path.getsize( self.filename )
780 currentSize = self.fileSize - self.__fp.tell()
718 781 neededSize = self.m_ProcessingHeader.blockSize + self.basicHeaderSize
719 782
720 783 if ( currentSize >= neededSize ):
@@ -731,7 +794,7 class SpectraReader( DataReader ):
731 794
732 795 if deltaTime > self.__maxTimeStep:
733 796 self.flagResetProcessing = 1
734 self.nReadBlocks = 0
797 #self.nReadBlocks = 0
735 798
736 799 return 1
737 800
@@ -753,11 +816,15 class SpectraReader( DataReader ):
753 816 self.__data_spc
754 817 self.__data_cspc
755 818 self.__data_dc
819
820 Exceptions:
821 Si un bloque leido no es un bloque valido
756 822 """
757 self.datablock_id = 0
758 self.__flagIsNewFile = 0
759 self.flagIsNewBlock = 1
823 #self.datablock_id = 0
824 #self.__flagIsNewFile = 0
825 #self.flagIsNewBlock = 1
760 826
827 blockOk_flag = False
761 828 fpointer = self.__fp.tell()
762 829
763 830 spc = numpy.fromfile( self.__fp, self.__dataType[0], self.__pts2read_SelfSpectra )
@@ -766,26 +833,36 class SpectraReader( DataReader ):
766 833
767 834 if self.online:
768 835 if (spc.size + cspc.size + dc.size) != self.__blocksize:
769 nTries = 0
770 while( nTries < self.__nTries ):
771 nTries += 1
772 print "Waiting for the next block, try %03d ..." % nTries
836 for nTries in range( self.__nTries ):
837 #nTries = 0
838 #while( nTries < self.__nTries ):
839 #nTries += 1
840 print "\tWaiting for the next block, try %03d ..." % (nTries+1)
773 841 time.sleep( self.__delay )
774 842 self.__fp.seek( fpointer )
775 843 fpointer = self.__fp.tell()
776 844 spc = numpy.fromfile( self.__fp, self.__dataType[0], self.__pts2read_SelfSpectra )
777 845 cspc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_CrossSpectra )
778 846 dc = numpy.fromfile( self.__fp, self.__dataType, self.__pts2read_DCchannels ) #int(self.m_ProcessingHeader.numHeights*self.m_SystemHeader.numChannels) )
847
779 848 if (spc.size + cspc.size + dc.size) == self.__blocksize:
780 nTries = 0
849 blockOk_flag = True
781 850 break
782 if nTries > 0:
783 return
851 #if (spc.size + cspc.size + dc.size) == self.__blocksize:
852 # nTries = 0
853 # break
854 if not( blockOk_flag ):
855 return 0
856 #if nTries > 0:
857 # return 0
784 858
859 try:
785 860 spc = spc.reshape( (self.nChannels, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
786
787 861 cspc = cspc.reshape( (self.nPairs, self.m_ProcessingHeader.numHeights, self.m_ProcessingHeader.profilesPerBlock) ) #transforma a un arreglo 3D
788 862 dc = dc.reshape( (self.m_SystemHeader.numChannels, self.m_ProcessingHeader.numHeights) ) #transforma a un arreglo 2D
863 except:
864 print "Data file %s is invalid" % self.filename
865 return 0
789 866
790 867 if not( self.m_ProcessingHeader.shif_fft ):
791 868 spc = numpy.roll( spc, self.m_ProcessingHeader.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
@@ -799,13 +876,14 class SpectraReader( DataReader ):
799 876 self.__data_cspc = cspc['real'] + cspc['imag']*1j
800 877 self.__data_dc = dc['real'] + dc['imag']*1j
801 878
879 self.datablock_id = 0
802 880 self.__flagIsNewFile = 0
803
804 881 self.flagIsNewBlock = 1
805 882
806 883 self.nReadBlocks += 1
807 self.datablock_id = 0
884 self.nBlocks += 1
808 885
886 return 1
809 887
810 888
811 889 def __hasNotDataInBuffer( self ):
@@ -830,7 +908,6 class SpectraReader( DataReader ):
830 908 filename : el ultimo file de una determinada carpeta
831 909 directory : eL directorio donde esta el file encontrado
832 910 """
833
834 911 print "Searching files ..."
835 912
836 913 dirList = []
@@ -843,7 +920,7 class SpectraReader( DataReader ):
843 920
844 921 dirList = sorted( dirList, key=str.lower ) #para que quede ordenado al margen de si el nombre esta en mayusculas o minusculas, utilizo la funcion sorted
845 922 if len(dirList) > 0 :
846 directory = dirList[-1]
923 directory = dirList[-1] #me quedo con el ultimo directorio de una carpeta
847 924 else:
848 925 year = startDateTime.timetuple().tm_year
849 926 doy = startDateTime.timetuple().tm_yday
@@ -916,7 +993,6 class SpectraReader( DataReader ):
916 993 de leer todos los bloques de datos de un determinado
917 994 archivo se pasa al siguiente archivo de la lista.
918 995 """
919
920 996 print "Searching files ..."
921 997
922 998 dirList = []
@@ -975,13 +1051,7 class SpectraReader( DataReader ):
975 1051
976 1052 file = os.path.join( path, dirfilename, filename )
977 1053
978 nTries = 0
979 while(True):
980
981 nTries += 1
982 if nTries > self.__nTries:
983 break
984
1054 for nTries in range( self.__nTries+1 ):
985 1055 try:
986 1056 fp = open( file,'rb' ) #lectura binaria
987 1057 except:
@@ -997,7 +1067,10 class SpectraReader( DataReader ):
997 1067 if m_BasicHeader.size > 24:
998 1068 break
999 1069
1000 print 'waiting for new block: try %02d' % ( nTries )
1070 if nTries >= self.__nTries: #si ya espero la cantidad de veces necesarias entonces ya no vuelve a esperar
1071 break
1072
1073 print '\twaiting for new block of file %s: try %02d' % ( file, nTries )
1001 1074 time.sleep( self.__delay)
1002 1075
1003 1076 if m_BasicHeader.size <= 24:
@@ -1049,26 +1122,31 class SpectraReader( DataReader ):
1049 1122 self.filenameList
1050 1123 self.online
1051 1124 """
1052
1053 1125 if online:
1054 nTries = 0
1055 while( nTries < self.__nTries ):
1056 nTries += 1
1126 fileOK_flag = False
1057 1127 subfolder = "D%04d%03d" % ( startDateTime.timetuple().tm_year, startDateTime.timetuple().tm_yday )
1058 year, doy, set, filename, dirfilename = self.__searchFilesOnLine( path, startDateTime, ext )
1059 if filename == None:
1060 1128 file = os.path.join( path, subfolder )
1061 print "Searching first file in \"%s\", try %03d ..." % ( file, nTries )
1062 time.sleep( self.__delay )
1063 else:
1129
1130 for nTries in range( self.__nTries+1 ): #espera por el 1er file
1131 year, doy, set, filename, dirfilename = self.__searchFilesOnLine( path, startDateTime, ext )
1132
1133 if filename != None:
1134 if isFileOK( os.path.join( path,dirfilename,filename ) ):
1135 fileOK_flag = True
1064 1136 break
1065 1137
1066 if filename == None:
1067 print "No files On Line"
1138 if nTries >= self.__nTries: #si ya espero la cantidad de veces necesarias entonces ya no vuelve a esperar
1139 break
1140
1141 print "Searching first file in \"%s\", try %03d ..." % ( file, nTries+1 )
1142 time.sleep( self.__delay )
1143
1144 if not( fileOK_flag ): #filename == None:
1145 print "No files on line or invalid first file"
1068 1146 return 0
1069 1147
1070 1148 if self.__initFilesOnline( path, dirfilename, filename ) == 0:
1071 print "The file %s hasn't enough data"
1149 print "The file %s hasn't enough data" % filename
1072 1150 return 0
1073 1151
1074 1152 self.__year = year
@@ -1128,7 +1206,8 class SpectraReader( DataReader ):
1128 1206 if not( self.__setNewBlock() ):
1129 1207 return 0
1130 1208
1131 self.__readBlock()
1209 if not( self.__readBlock() ):
1210 return 0
1132 1211
1133 1212 self.__lastUTTime = self.m_BasicHeader.utc
1134 1213
@@ -1157,7 +1236,9 class SpectraReader( DataReader ):
1157 1236
1158 1237 if self.__hasNotDataInBuffer():
1159 1238
1160 self.readNextBlock()
1239 if not( self.readNextBlock() ):
1240 self.__setNextFile()
1241 return 0
1161 1242
1162 1243 self.m_Spectra.m_BasicHeader = self.m_BasicHeader.copy()
1163 1244 self.m_Spectra.m_ProcessingHeader = self.m_ProcessingHeader.copy()
@@ -1172,6 +1253,10 class SpectraReader( DataReader ):
1172 1253
1173 1254 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
1174 1255
1256 if self.__data_dc == None:
1257 self.m_Voltage.flagNoData = True
1258 return 0
1259
1175 1260 self.m_Spectra.flagNoData = False
1176 1261 self.m_Spectra.flagResetProcessing = self.flagResetProcessing
1177 1262
@@ -1267,6 +1352,7 class SpectraWriter( DataWriter ):
1267 1352 self.__wrProcessingHeader()
1268 1353 self.__dataType = self.m_Spectra.dataType
1269 1354
1355
1270 1356 def __writeBasicHeader( self, fp=None ):
1271 1357 """
1272 1358 Escribe solo el Basic header en el file creado
@@ -1279,6 +1365,7 class SpectraWriter( DataWriter ):
1279 1365
1280 1366 self.m_BasicHeader.write(fp)
1281 1367
1368
1282 1369 def __wrSystemHeader( self, fp=None ):
1283 1370 """
1284 1371 Escribe solo el System header en el file creado
@@ -1291,6 +1378,7 class SpectraWriter( DataWriter ):
1291 1378
1292 1379 self.m_SystemHeader.write(fp)
1293 1380
1381
1294 1382 def __wrRadarControllerHeader( self, fp=None ):
1295 1383 """
1296 1384 Escribe solo el RadarController header en el file creado
@@ -1303,6 +1391,7 class SpectraWriter( DataWriter ):
1303 1391
1304 1392 self.m_RadarControllerHeader.write(fp)
1305 1393
1394
1306 1395 def __wrProcessingHeader( self, fp=None ):
1307 1396 """
1308 1397 Escribe solo el Processing header en el file creado
@@ -1315,6 +1404,7 class SpectraWriter( DataWriter ):
1315 1404
1316 1405 self.m_ProcessingHeader.write(fp)
1317 1406
1407
1318 1408 def __setNextFile( self ):
1319 1409 """
1320 1410 Determina el siguiente file que sera escrito
@@ -1336,7 +1426,7 class SpectraWriter( DataWriter ):
1336 1426 if self.__fp != None:
1337 1427 self.__fp.close()
1338 1428
1339 if self.m_BasicHeader.size <= 24: return 0 #no existe la suficiente data para ser escrita
1429 #if self.m_BasicHeader.size <= 24: return 0 #no existe la suficiente data para ser escrita
1340 1430
1341 1431 timeTuple = time.localtime(self.m_Spectra.m_BasicHeader.utc) # utc from m_Spectra
1342 1432 subfolder = 'D%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
@@ -1383,6 +1473,7 class SpectraWriter( DataWriter ):
1383 1473
1384 1474 return 1
1385 1475
1476
1386 1477 def __setNewBlock( self ):
1387 1478 """
1388 1479 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
@@ -1406,6 +1497,7 class SpectraWriter( DataWriter ):
1406 1497
1407 1498 return 1
1408 1499
1500
1409 1501 def __writeBlock( self ):
1410 1502 """
1411 1503 Escribe el buffer en el file designado
@@ -1448,11 +1540,8 class SpectraWriter( DataWriter ):
1448 1540 self.__data_dc.fill(0)
1449 1541
1450 1542 self.__flagIsNewFile = 0
1451
1452 1543 self.flagIsNewBlock = 1
1453
1454 1544 self.nWriteBlocks += 1
1455
1456 1545 self.__blocksCounter += 1
1457 1546
1458 1547
@@ -1468,7 +1557,6 class SpectraWriter( DataWriter ):
1468 1557 return 0
1469 1558
1470 1559 self.__writeBlock()
1471
1472 1560 return 1
1473 1561
1474 1562
@@ -1505,7 +1593,6 class SpectraWriter( DataWriter ):
1505 1593 self.__data_dc = self.m_Spectra.data_dc
1506 1594
1507 1595 if True:
1508 #time.sleep( 3 )
1509 1596 self.__getHeader()
1510 1597 self.writeNextBlock()
1511 1598
General Comments 0
You need to be logged in to leave comments. Login now