From 2ca28c2f318ecef481628d80d89b7e4cd61857c2 2023-09-14 16:32:49 From: Alexander Valdez Date: 2023-09-14 16:32:49 Subject: [PATCH] Update test Julia_EEJ.py --- diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index 7d17366..79c2fd4 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -384,7 +384,7 @@ def isRadarFolder(folder): def isRadarFile(file): - try: + try: year = int(file[1:5]) doy = int(file[5:8]) set = int(file[8:11]) @@ -395,10 +395,10 @@ def isRadarFile(file): def getDateFromRadarFile(file): - try: + try: year = int(file[1:5]) doy = int(file[5:8]) - set = int(file[8:11]) + set = int(file[8:11]) except: return None @@ -417,7 +417,7 @@ def getDateFromRadarFolder(folder): return thisDate def parse_format(s, fmt): - + for i in range(fmt.count('%')): x = fmt.index('%') d = DT_DIRECTIVES[fmt[x:x+2]] @@ -484,7 +484,7 @@ class Reader(object): def run(self): - raise NotImplementedError + raise NotImplementedError def getAllowedArgs(self): if hasattr(self, '__attrs__'): @@ -496,19 +496,19 @@ class Reader(object): for key, value in kwargs.items(): setattr(self, key, value) - + def find_folders(self, path, startDate, endDate, folderfmt, last=False): - folders = [x for f in path.split(',') + folders = [x for f in path.split(',') for x in os.listdir(f) if os.path.isdir(os.path.join(f, x))] folders.sort() if last: folders = [folders[-1]] - for folder in folders: - try: - dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date() + for folder in folders: + try: + dt = datetime.datetime.strptime(parse_format(folder, folderfmt), folderfmt).date() if dt >= startDate and dt <= endDate: yield os.path.join(path, folder) else: @@ -517,38 +517,37 @@ class Reader(object): log.log('Skiping folder {}'.format(folder), self.name) continue return - - def find_files(self, folders, ext, filefmt, startDate=None, endDate=None, + + def find_files(self, folders, ext, filefmt, startDate=None, endDate=None, expLabel='', last=False): - - for path in folders: - files = glob.glob1(path, '*{}'.format(ext)) + for path in folders: + files = glob.glob1(path+'/'+expLabel, '*{}'.format(ext)) files.sort() if last: - if files: + if files: fo = files[-1] - try: + try: dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date() - yield os.path.join(path, expLabel, fo) - except Exception as e: + yield os.path.join(path, expLabel, fo) + except Exception as e: pass return else: return for fo in files: - try: - dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date() + try: + dt = datetime.datetime.strptime(parse_format(fo, filefmt), filefmt).date() if dt >= startDate and dt <= endDate: yield os.path.join(path, expLabel, fo) else: log.log('Skiping file {}'.format(fo), self.name) except Exception as e: log.log('Skiping file {}'.format(fo), self.name) - continue + continue def searchFilesOffLine(self, path, startDate, endDate, - expLabel, ext, walk, + expLabel, ext, walk, filefmt, folderfmt): """Search files in offline mode for the given arguments @@ -563,10 +562,10 @@ class Reader(object): folders = path.split(',') return self.find_files( - folders, ext, filefmt, startDate, endDate, expLabel) + folders, ext, filefmt, startDate, endDate, expLabel) def searchFilesOnLine(self, path, startDate, endDate, - expLabel, ext, walk, + expLabel, ext, walk, filefmt, folderfmt): """Search for the last file of the last folder @@ -579,13 +578,13 @@ class Reader(object): Return: generator with the full path of last filename """ - + if walk: folders = self.find_folders( path, startDate, endDate, folderfmt, last=True) else: folders = path.split(',') - + return self.find_files( folders, ext, filefmt, startDate, endDate, expLabel, last=True) @@ -594,13 +593,12 @@ class Reader(object): while True: if self.fp != None: - self.fp.close() + self.fp.close() if self.online: newFile = self.setNextFileOnline() else: newFile = self.setNextFileOffline() - if not(newFile): if self.online: raise schainpy.admin.SchainError('Time to wait for new files reach') @@ -609,10 +607,10 @@ class Reader(object): raise schainpy.admin.SchainWarning('No files found in the given path') else: raise schainpy.admin.SchainWarning('No more files to read') - + if self.verifyFile(self.filename): break - + log.log('Opening file: %s' % self.filename, self.name) self.readFirstHeader() @@ -625,15 +623,16 @@ class Reader(object): self.filename self.fp self.filesize - + Return: boolean """ + nextFile = True nextDay = False - for nFiles in range(self.nFiles+1): + for nFiles in range(self.nFiles+1): for nTries in range(self.nTries): fullfilename, filename = self.checkForRealPath(nextFile, nextDay) if fullfilename is not None: @@ -643,18 +642,18 @@ class Reader(object): self.name) time.sleep(self.delay) nextFile = False - continue - + continue + if fullfilename is not None: break - + self.nTries = 1 - nextFile = True + nextFile = True if nFiles == (self.nFiles - 1): log.log('Trying with next day...', self.name) nextDay = True - self.nTries = 3 + self.nTries = 3 if fullfilename: self.fileSize = os.path.getsize(fullfilename) @@ -666,18 +665,18 @@ class Reader(object): self.flagNoMoreFiles = 0 self.fileIndex += 1 return 1 - else: + else: return 0 - + def setNextFileOffline(self): """Open the next file to be readed in offline mode""" - + try: filename = next(self.filenameList) self.fileIndex +=1 except StopIteration: self.flagNoMoreFiles = 1 - return 0 + return 0 self.filename = filename self.fileSize = os.path.getsize(filename) @@ -685,22 +684,22 @@ class Reader(object): self.flagIsNewFile = 1 return 1 - + @staticmethod def isDateTimeInRange(dt, startDate, endDate, startTime, endTime): """Check if the given datetime is in range""" - + if startDate <= dt.date() <= endDate: if startTime <= dt.time() <= endTime: return True return False - + def verifyFile(self, filename): """Check for a valid file - + Arguments: filename -- full path filename - + Return: boolean """ @@ -711,7 +710,7 @@ class Reader(object): """Check if the next file to be readed exists""" raise NotImplementedError - + def readFirstHeader(self): """Parse the file header""" @@ -783,8 +782,7 @@ class JRODataReader(Reader): Return: str -- fullpath of the file """ - - + if nextFile: self.set += 1 if nextDay: @@ -796,7 +794,7 @@ class JRODataReader(Reader): prefixFileList = ['d', 'D'] elif self.ext.lower() == ".pdata": # spectra prefixFileList = ['p', 'P'] - + # barrido por las combinaciones posibles for prefixDir in prefixDirList: thispath = self.path @@ -816,9 +814,9 @@ class JRODataReader(Reader): if os.path.exists(fullfilename): return fullfilename, filename - - return None, filename - + + return None, filename + def __waitNewBlock(self): """ Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. @@ -860,9 +858,9 @@ class JRODataReader(Reader): def __setNewBlock(self): if self.fp == None: - return 0 - - if self.flagIsNewFile: + return 0 + + if self.flagIsNewFile: self.lastUTTime = self.basicHeaderObj.utc return 1 @@ -875,12 +873,12 @@ class JRODataReader(Reader): currentSize = self.fileSize - self.fp.tell() neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize - + if (currentSize >= neededSize): self.basicHeaderObj.read(self.fp) self.lastUTTime = self.basicHeaderObj.utc return 1 - + if self.__waitNewBlock(): self.lastUTTime = self.basicHeaderObj.utc return 1 @@ -892,7 +890,6 @@ class JRODataReader(Reader): self.lastUTTime = self.basicHeaderObj.utc self.flagDiscontinuousBlock = 0 - if deltaTime > self.maxTimeStep: self.flagDiscontinuousBlock = 1 @@ -966,10 +963,10 @@ class JRODataReader(Reader): except IOError: log.error("File {} can't be opened".format(filename), self.name) return False - + if self.online and self.waitDataBlock(0): pass - + basicHeaderObj = BasicHeader(LOCALTIME) systemHeaderObj = SystemHeader() radarControllerHeaderObj = RadarControllerHeader() @@ -996,7 +993,7 @@ class JRODataReader(Reader): dt2 = basicHeaderObj.datatime if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \ self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime): - flag = False + flag = False fp.close() return flag @@ -1105,11 +1102,11 @@ class JRODataReader(Reader): return dateList def setup(self, **kwargs): - + self.set_kwargs(**kwargs) if not self.ext.startswith('.'): self.ext = '.{}'.format(self.ext) - + if self.server is not None: if 'tcp://' in self.server: address = server @@ -1131,36 +1128,36 @@ class JRODataReader(Reader): for nTries in range(self.nTries): fullpath = self.searchFilesOnLine(self.path, self.startDate, - self.endDate, self.expLabel, self.ext, self.walk, + self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) try: fullpath = next(fullpath) except: fullpath = None - + if fullpath: break log.warning( 'Waiting {} sec for a valid file in {}: try {} ...'.format( - self.delay, self.path, nTries + 1), + self.delay, self.path, nTries + 1), self.name) time.sleep(self.delay) if not(fullpath): raise schainpy.admin.SchainError( - 'There isn\'t any valid file in {}'.format(self.path)) + 'There isn\'t any valid file in {}'.format(self.path)) pathname, filename = os.path.split(fullpath) self.year = int(filename[1:5]) self.doy = int(filename[5:8]) - self.set = int(filename[8:11]) - 1 + self.set = int(filename[8:11]) - 1 else: log.log("Searching files in {}".format(self.path), self.name) - self.filenameList = self.searchFilesOffLine(self.path, self.startDate, + self.filenameList = self.searchFilesOffLine(self.path, self.startDate, self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt) - + self.setNextFile() return @@ -1181,7 +1178,7 @@ class JRODataReader(Reader): self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs - + def getFirstHeader(self): raise NotImplementedError @@ -1214,8 +1211,8 @@ class JRODataReader(Reader): """ Arguments: - path : - startDate : + path : + startDate : endDate : startTime : endTime : @@ -1284,7 +1281,7 @@ class JRODataWriter(Reader): dtype_width = get_dtype_width(dtype_index) return dtype_width - + def getProcessFlags(self): processFlags = 0 @@ -1322,9 +1319,9 @@ class JRODataWriter(Reader): self.basicHeaderObj.size = self.basicHeaderSize # bytes self.basicHeaderObj.version = self.versionFile - self.basicHeaderObj.dataBlock = self.nTotalBlocks + self.basicHeaderObj.dataBlock = self.nTotalBlocks utc = numpy.floor(self.dataOut.utctime) - milisecond = (self.dataOut.utctime - utc) * 1000.0 + milisecond = (self.dataOut.utctime - utc) * 1000.0 self.basicHeaderObj.utc = utc self.basicHeaderObj.miliSecond = milisecond self.basicHeaderObj.timeZone = self.dataOut.timeZone @@ -1465,9 +1462,9 @@ class JRODataWriter(Reader): if self.dataOut.datatime.date() > self.fileDate: setFile = 0 self.nTotalBlocks = 0 - + filen = '{}{:04d}{:03d}{:03d}{}'.format( - self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) + self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) filename = os.path.join(path, subfolder, filen) @@ -1515,11 +1512,11 @@ class JRODataWriter(Reader): self.ext = ext.lower() self.path = path - + if set is None: self.setFile = -1 else: - self.setFile = set - 1 + self.setFile = set - 1 self.blocksPerFile = blocksPerFile self.profilesPerBlock = profilesPerBlock @@ -1572,4 +1569,4 @@ class printInfo(Operation): else: log.warning('Header {} Not found in object'.format(header)) - self.__printInfo = False + self.__printInfo = False \ No newline at end of file diff --git a/schainpy/model/proc/jroproc_spectra.py b/schainpy/model/proc/jroproc_spectra.py index e96c844..651c0c6 100644 --- a/schainpy/model/proc/jroproc_spectra.py +++ b/schainpy/model/proc/jroproc_spectra.py @@ -430,7 +430,6 @@ class SpectraProc(ProcessingUnit): return 1 - class GetSNR(Operation): ''' Written by R. Flores @@ -535,17 +534,17 @@ class removeInterference(Operation): for i in range(cspc.shape[0]): LinePower= numpy.sum(realCspc[i], axis=0) Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)] - SelectedHeights = Heights[ numpy.where(LinePower < Threshold) ] - InterferenceSum = numpy.sum(realCspc[i,:,SelectedHeights],axis=0) + SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ] + InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 ) InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)] InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)] - - - InterferenceRange = numpy.where(([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) ) + + + InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) ) #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax])) if len(InterferenceRange)