##// END OF EJS Templates
schain
RSS
Update test Julia_EEJ.py
Alexander Valdez -
r1685:2ca28c2f318e
parent child
Show More
Side by Side Unified
Context file:
r1685:2ca28c2f318e Loading diff...:
Show file before | Show file after | Hide comments
@@ -384,7 +384,7 def isRadarFolder(folder):
384 384
385 385
386 386 def isRadarFile(file):
387 try:
387 try:
388 388 year = int(file[1:5])
389 389 doy = int(file[5:8])
390 390 set = int(file[8:11])
@@ -395,10 +395,10 def isRadarFile(file):
395 395
396 396
397 397 def getDateFromRadarFile(file):
398 try:
398 try:
399 399 year = int(file[1:5])
400 400 doy = int(file[5:8])
401 set = int(file[8:11])
401 set = int(file[8:11])
402 402 except:
403 403 return None
404 404
@@ -417,7 +417,7 def getDateFromRadarFolder(folder):
417 417 return thisDate
418 418
419 419 def parse_format(s, fmt):
420
420
421 421 for i in range(fmt.count('%')):
422 422 x = fmt.index('%')
423 423 d = DT_DIRECTIVES[fmt[x:x+2]]
@@ -484,7 +484,7 class Reader(object):
484 484
485 485 def run(self):
486 486
487 raise NotImplementedError
487 raise NotImplementedError
488 488
489 489 def getAllowedArgs(self):
490 490 if hasattr(self, '__attrs__'):
@@ -496,19 +496,19 class Reader(object):
496 496
497 497 for key, value in kwargs.items():
498 498 setattr(self, key, value)
499
499
500 500 def find_folders(self, path, startDate, endDate, folderfmt, last=False):
501 501
502 folders = [x for f in path.split(',')
502 folders = [x for f in path.split(',')
503 503 for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))]
504 504 folders.sort()
505 505
506 506 if last:
507 507 folders = [folders[-1]]
508 508
509 for folder in folders:
510 try:
511 dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
509 for folder in folders:
510 try:
511 dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date()
512 512 if dt >= startDate and dt <= endDate:
513 513 yield os.path.join(path, folder)
514 514 else:
@@ -517,38 +517,37 class Reader(object):
517 517 log.log('Skiping folder {}'.format(folder), self.name)
518 518 continue
519 519 return
520
521 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
520
521 def find_files(self, folders, ext, filefmt, startDate=None, endDate=None,
522 522 expLabel='', last=False):
523
524 for path in folders:
525 files = glob.glob1(path, '*{}'.format(ext))
523 for path in folders:
524 files = glob.glob1(path+'/'+expLabel, '*{}'.format(ext))
526 525 files.sort()
527 526 if last:
528 if files:
527 if files:
529 528 fo = files[-1]
530 try:
529 try:
531 530 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
532 yield os.path.join(path, expLabel, fo)
533 except Exception as e:
531 yield os.path.join(path, expLabel, fo)
532 except Exception as e:
534 533 pass
535 534 return
536 535 else:
537 536 return
538 537
539 538 for fo in files:
540 try:
541 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
539 try:
540 dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date()
542 541 if dt >= startDate and dt <= endDate:
543 542 yield os.path.join(path, expLabel, fo)
544 543 else:
545 544 log.log('Skiping file {}'.format(fo), self.name)
546 545 except Exception as e:
547 546 log.log('Skiping file {}'.format(fo), self.name)
548 continue
547 continue
549 548
550 549 def searchFilesOffLine(self, path, startDate, endDate,
551 expLabel, ext, walk,
550 expLabel, ext, walk,
552 551 filefmt, folderfmt):
553 552 """Search files in offline mode for the given arguments
554 553
@@ -563,10 +562,10 class Reader(object):
563 562 folders = path.split(',')
564 563
565 564 return self.find_files(
566 folders, ext, filefmt, startDate, endDate, expLabel)
565 folders, ext, filefmt, startDate, endDate, expLabel)
567 566
568 567 def searchFilesOnLine(self, path, startDate, endDate,
569 expLabel, ext, walk,
568 expLabel, ext, walk,
570 569 filefmt, folderfmt):
571 570 """Search for the last file of the last folder
572 571
@@ -579,13 +578,13 class Reader(object):
579 578 Return:
580 579 generator with the full path of last filename
581 580 """
582
581
583 582 if walk:
584 583 folders = self.find_folders(
585 584 path, startDate, endDate, folderfmt, last=True)
586 585 else:
587 586 folders = path.split(',')
588
587
589 588 return self.find_files(
590 589 folders, ext, filefmt, startDate, endDate, expLabel, last=True)
591 590
@@ -594,13 +593,12 class Reader(object):
594 593
595 594 while True:
596 595 if self.fp != None:
597 self.fp.close()
596 self.fp.close()
598 597
599 598 if self.online:
600 599 newFile = self.setNextFileOnline()
601 600 else:
602 601 newFile = self.setNextFileOffline()
603
604 602 if not(newFile):
605 603 if self.online:
606 604 raise schainpy.admin.SchainError('Time to wait for new files reach')
@@ -609,10 +607,10 class Reader(object):
609 607 raise schainpy.admin.SchainWarning('No files found in the given path')
610 608 else:
611 609 raise schainpy.admin.SchainWarning('No more files to read')
612
610
613 611 if self.verifyFile(self.filename):
614 612 break
615
613
616 614 log.log('Opening file: %s' % self.filename, self.name)
617 615
618 616 self.readFirstHeader()
@@ -625,15 +623,16 class Reader(object):
625 623 self.filename
626 624 self.fp
627 625 self.filesize
628
626
629 627 Return:
630 628 boolean
631 629
632 630 """
631
633 632 nextFile = True
634 633 nextDay = False
635 634
636 for nFiles in range(self.nFiles+1):
635 for nFiles in range(self.nFiles+1):
637 636 for nTries in range(self.nTries):
638 637 fullfilename, filename = self.checkForRealPath(nextFile, nextDay)
639 638 if fullfilename is not None:
@@ -643,18 +642,18 class Reader(object):
643 642 self.name)
644 643 time.sleep(self.delay)
645 644 nextFile = False
646 continue
647
645 continue
646
648 647 if fullfilename is not None:
649 648 break
650
649
651 650 self.nTries = 1
652 nextFile = True
651 nextFile = True
653 652
654 653 if nFiles == (self.nFiles - 1):
655 654 log.log('Trying with next day...', self.name)
656 655 nextDay = True
657 self.nTries = 3
656 self.nTries = 3
658 657
659 658 if fullfilename:
660 659 self.fileSize = os.path.getsize(fullfilename)
@@ -666,18 +665,18 class Reader(object):
666 665 self.flagNoMoreFiles = 0
667 666 self.fileIndex += 1
668 667 return 1
669 else:
668 else:
670 669 return 0
671
670
672 671 def setNextFileOffline(self):
673 672 """Open the next file to be readed in offline mode"""
674
673
675 674 try:
676 675 filename = next(self.filenameList)
677 676 self.fileIndex +=1
678 677 except StopIteration:
679 678 self.flagNoMoreFiles = 1
680 return 0
679 return 0
681 680
682 681 self.filename = filename
683 682 self.fileSize = os.path.getsize(filename)
@@ -685,22 +684,22 class Reader(object):
685 684 self.flagIsNewFile = 1
686 685
687 686 return 1
688
687
689 688 @staticmethod
690 689 def isDateTimeInRange(dt, startDate, endDate, startTime, endTime):
691 690 """Check if the given datetime is in range"""
692
691
693 692 if startDate <= dt.date() <= endDate:
694 693 if startTime <= dt.time() <= endTime:
695 694 return True
696 695 return False
697
696
698 697 def verifyFile(self, filename):
699 698 """Check for a valid file
700
699
701 700 Arguments:
702 701 filename -- full path filename
703
702
704 703 Return:
705 704 boolean
706 705 """
@@ -711,7 +710,7 class Reader(object):
711 710 """Check if the next file to be readed exists"""
712 711
713 712 raise NotImplementedError
714
713
715 714 def readFirstHeader(self):
716 715 """Parse the file header"""
717 716
@@ -783,8 +782,7 class JRODataReader(Reader):
783 782 Return:
784 783 str -- fullpath of the file
785 784 """
786
787
785
788 786 if nextFile:
789 787 self.set += 1
790 788 if nextDay:
@@ -796,7 +794,7 class JRODataReader(Reader):
796 794 prefixFileList = ['d', 'D']
797 795 elif self.ext.lower() == ".pdata": # spectra
798 796 prefixFileList = ['p', 'P']
799
797
800 798 # barrido por las combinaciones posibles
801 799 for prefixDir in prefixDirList:
802 800 thispath = self.path
@@ -816,9 +814,9 class JRODataReader(Reader):
816 814
817 815 if os.path.exists(fullfilename):
818 816 return fullfilename, filename
819
820 return None, filename
821
817
818 return None, filename
819
822 820 def __waitNewBlock(self):
823 821 """
824 822 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
@@ -860,9 +858,9 class JRODataReader(Reader):
860 858 def __setNewBlock(self):
861 859
862 860 if self.fp == None:
863 return 0
864
865 if self.flagIsNewFile:
861 return 0
862
863 if self.flagIsNewFile:
866 864 self.lastUTTime = self.basicHeaderObj.utc
867 865 return 1
868 866
@@ -875,12 +873,12 class JRODataReader(Reader):
875 873
876 874 currentSize = self.fileSize - self.fp.tell()
877 875 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
878
876
879 877 if (currentSize >= neededSize):
880 878 self.basicHeaderObj.read(self.fp)
881 879 self.lastUTTime = self.basicHeaderObj.utc
882 880 return 1
883
881
884 882 if self.__waitNewBlock():
885 883 self.lastUTTime = self.basicHeaderObj.utc
886 884 return 1
@@ -892,7 +890,6 class JRODataReader(Reader):
892 890 self.lastUTTime = self.basicHeaderObj.utc
893 891
894 892 self.flagDiscontinuousBlock = 0
895
896 893 if deltaTime > self.maxTimeStep:
897 894 self.flagDiscontinuousBlock = 1
898 895
@@ -966,10 +963,10 class JRODataReader(Reader):
966 963 except IOError:
967 964 log.error("File {} can't be opened".format(filename), self.name)
968 965 return False
969
966
970 967 if self.online and self.waitDataBlock(0):
971 968 pass
972
969
973 970 basicHeaderObj = BasicHeader(LOCALTIME)
974 971 systemHeaderObj = SystemHeader()
975 972 radarControllerHeaderObj = RadarControllerHeader()
@@ -996,7 +993,7 class JRODataReader(Reader):
996 993 dt2 = basicHeaderObj.datatime
997 994 if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \
998 995 self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime):
999 flag = False
996 flag = False
1000 997
1001 998 fp.close()
1002 999 return flag
@@ -1105,11 +1102,11 class JRODataReader(Reader):
1105 1102 return dateList
1106 1103
1107 1104 def setup(self, **kwargs):
1108
1105
1109 1106 self.set_kwargs(**kwargs)
1110 1107 if not self.ext.startswith('.'):
1111 1108 self.ext = '.{}'.format(self.ext)
1112
1109
1113 1110 if self.server is not None:
1114 1111 if 'tcp://' in self.server:
1115 1112 address = server
@@ -1131,36 +1128,36 class JRODataReader(Reader):
1131 1128
1132 1129 for nTries in range(self.nTries):
1133 1130 fullpath = self.searchFilesOnLine(self.path, self.startDate,
1134 self.endDate, self.expLabel, self.ext, self.walk,
1131 self.endDate, self.expLabel, self.ext, self.walk,
1135 1132 self.filefmt, self.folderfmt)
1136 1133
1137 1134 try:
1138 1135 fullpath = next(fullpath)
1139 1136 except:
1140 1137 fullpath = None
1141
1138
1142 1139 if fullpath:
1143 1140 break
1144 1141
1145 1142 log.warning(
1146 1143 'Waiting {} sec for a valid file in {}: try {} ...'.format(
1147 self.delay, self.path, nTries + 1),
1144 self.delay, self.path, nTries + 1),
1148 1145 self.name)
1149 1146 time.sleep(self.delay)
1150 1147
1151 1148 if not(fullpath):
1152 1149 raise schainpy.admin.SchainError(
1153 'There isn\'t any valid file in {}'.format(self.path))
1150 'There isn\'t any valid file in {}'.format(self.path))
1154 1151
1155 1152 pathname, filename = os.path.split(fullpath)
1156 1153 self.year = int(filename[1:5])
1157 1154 self.doy = int(filename[5:8])
1158 self.set = int(filename[8:11]) - 1
1155 self.set = int(filename[8:11]) - 1
1159 1156 else:
1160 1157 log.log("Searching files in {}".format(self.path), self.name)
1161 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
1158 self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
1162 1159 self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
1163
1160
1164 1161 self.setNextFile()
1165 1162
1166 1163 return
@@ -1181,7 +1178,7 class JRODataReader(Reader):
1181 1178 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1182 1179
1183 1180 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1184
1181
1185 1182 def getFirstHeader(self):
1186 1183
1187 1184 raise NotImplementedError
@@ -1214,8 +1211,8 class JRODataReader(Reader):
1214 1211 """
1215 1212
1216 1213 Arguments:
1217 path :
1218 startDate :
1214 path :
1215 startDate :
1219 1216 endDate :
1220 1217 startTime :
1221 1218 endTime :
@@ -1284,7 +1281,7 class JRODataWriter(Reader):
1284 1281 dtype_width = get_dtype_width(dtype_index)
1285 1282
1286 1283 return dtype_width
1287
1284
1288 1285 def getProcessFlags(self):
1289 1286
1290 1287 processFlags = 0
@@ -1322,9 +1319,9 class JRODataWriter(Reader):
1322 1319
1323 1320 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1324 1321 self.basicHeaderObj.version = self.versionFile
1325 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1322 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1326 1323 utc = numpy.floor(self.dataOut.utctime)
1327 milisecond = (self.dataOut.utctime - utc) * 1000.0
1324 milisecond = (self.dataOut.utctime - utc) * 1000.0
1328 1325 self.basicHeaderObj.utc = utc
1329 1326 self.basicHeaderObj.miliSecond = milisecond
1330 1327 self.basicHeaderObj.timeZone = self.dataOut.timeZone
@@ -1465,9 +1462,9 class JRODataWriter(Reader):
1465 1462 if self.dataOut.datatime.date() > self.fileDate:
1466 1463 setFile = 0
1467 1464 self.nTotalBlocks = 0
1468
1465
1469 1466 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1470 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1467 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1471 1468
1472 1469 filename = os.path.join(path, subfolder, filen)
1473 1470
@@ -1515,11 +1512,11 class JRODataWriter(Reader):
1515 1512 self.ext = ext.lower()
1516 1513
1517 1514 self.path = path
1518
1515
1519 1516 if set is None:
1520 1517 self.setFile = -1
1521 1518 else:
1522 self.setFile = set - 1
1519 self.setFile = set - 1
1523 1520
1524 1521 self.blocksPerFile = blocksPerFile
1525 1522 self.profilesPerBlock = profilesPerBlock
@@ -1572,4 +1569,4 class printInfo(Operation):
1572 1569 else:
1573 1570 log.warning('Header {} Not found in object'.format(header))
1574 1571
1575 self.__printInfo = False
1572 self.__printInfo = False No newline at end of file
Show file before | Show file after | Hide comments
@@ -430,7 +430,6 class SpectraProc(ProcessingUnit):
430 430
431 431 return 1
432 432
433
434 433 class GetSNR(Operation):
435 434 '''
436 435 Written by R. Flores
@@ -535,17 +534,17 class removeInterference(Operation):
535 534 for i in range(cspc.shape[0]):
536 535 LinePower= numpy.sum(realCspc[i], axis=0)
537 536 Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
538 SelectedHeights = Heights[ numpy.where(LinePower < Threshold) ]
539 InterferenceSum = numpy.sum(realCspc[i,:,SelectedHeights],axis=0)
537 SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
538 InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
540 539 InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
541 540 InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
542
543
544 InterferenceRange = numpy.where(([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
541
542
543 InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
545 544 #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
546 545 if len(InterferenceRange)<int(cspc.shape[1]*0.3):
547 546 cspc[i,InterferenceRange,:] = numpy.NaN
548
547
549 548 self.dataOut.data_cspc = cspc
550 549
551 550 def removeInterference(self, interf=2, hei_interf=None, nhei_interf=None, offhei_interf=None):
Show file before | Show file after | Hide comments
@@ -74,7 +74,7 if __name__ == '__main__':
74 74 walk=1,
75 75 expLabel='150EEJ',
76 76 getByBlock=1,
77 delay=20)
77 delay=20)
78 78
79 79 # opObj00 = readUnitConfObj.addOperation(name='printInfo')
80 80 # opObj00 = readUnitConfObj.addOperation(name='printNumberOfBlock')
General Comments 0
You need to be logged in to leave comments. Login now