schain Commit - r1373:fa4c334951aa · Jicamarca Repository

Cambio en la lectura de archivos, se incluyó detección de cambio de experimento

joabAM -

r1373:fa4c334951aa

parent child

Context file:

r1373:fa4c334951aa

Collapse all files

schainpy/model/io/jroIO_kamisr.py +26 -24

             '''
             Created on Set 9, 2015
             @author: roj-idl71 Karim Kuyeng
             '''
             import os
             import sys
             import glob
             import fnmatch
             import datetime
             import time
             import re
             import h5py
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from numpy import imag
             class AMISRReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.set = None
                     self.subset = None
                     self.extension_file = '.h5'
                     self.dtc_str = 'dtc'
                     self.dtc_id = 0
                     self.status = True
                     self.isConfig = False
                     self.dirnameList = []
                     self.filenameList = []
                     self.fileIndex = None
                     self.flagNoMoreFiles = False
                     self.flagIsNewFile = 0
                     self.filename = ''
                     self.amisrFilePointer = None
                     self.realBeamCode = []
-                    #self.dataset = None
+                    self.dataShape = None
                     self.profileIndex = 0
                     self.beamCodeByFrame = None
                     self.radacTimeByFrame = None
                     self.dataset = None
                     self.__firstFile = True
                     self.buffer = None
                     self.timezone = 'ut'
                     self.__waitForNewFile = 20
                     self.__filename_online = None
                     #Is really necessary create the output object in the initializer
                     self.dataOut = Voltage()
                     self.dataOut.error=False
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=None,
                                 endTime=None,
                                 walk=True,
                                 timezone='ut',
                                 all=0,
                                 code = None,
                                 nCode = 0,
                                 nBaud = 0,
                                 online=False):
                     self.timezone = timezone
                     self.all = all
                     self.online = online
                     self.code = code
                     self.nCode = int(nCode)
                     self.nBaud = int(nBaud)
                     #self.findFiles()
                     if not(online):
                         #Busqueda de archivos offline
                         self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
                     else:
                         self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
                     if not(self.filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
-                    self.fileIndex = -1
+                    self.fileIndex = 0
                     self.readNextFile(online)
                     '''
                     Add code
                     '''
                     self.isConfig = True
                     # print("Setup Done")
                     pass
                 def readAMISRHeader(self,fp):
+                    if self.isConfig :
+                        newShape = fp.get('Raw11/Data/Samples/Data').shape
+                        if self.dataShape != newShape:
+                            print("\nERROR, NEW FILE IS A DIFFERENT EXPERIMENT \n")
+                            return 0
+                    else:
+                        self.dataShape = fp.get('Raw11/Data/Samples/Data').shape
                     header = 'Raw11/Data/RadacHeader'
                     self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
                     if (self.startDate> datetime.date(2021, 7, 15)): #Se cambió la forma de extracción de Apuntes el 17
                         self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
                         self.trueBeams = self.beamcodeFile.split("\n")
                         self.trueBeams.pop()#remove last
                         [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
                         self.beamCode = [int(x, 16) for x in self.realBeamCode]
                     else:
                         _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
                         self.beamCode = _beamCode[0,:]
                     #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
                     self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
                     self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
                     self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
                     self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
                     self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
                     self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
                     self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
                     self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
                     self.frequency =  fp.get('Rx/Frequency')
                     txAus = fp.get('Raw11/Data/Pulsewidth')
                     self.nblocks = self.pulseCount.shape[0] #nblocks
                     self.nprofiles = self.pulseCount.shape[1] #nprofile
                     self.nsa = self.nsamplesPulse[0,0] #ngates
                     self.nchannels = len(self.beamCode)
                     self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
                     #self.__waitForNewFile = self.nblocks  # wait depending on the number of blocks since each block is 1 sec
                     self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
                     #filling radar controller header parameters
                     self.__ippKm = self.ippSeconds *.15*1e6 # in km
                     self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
                     self.__txB = 0
                     nWindows=1
                     self.__nSamples = self.nsa
                     self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
                     self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
                     #for now until understand why the code saved is different (code included even though code not in tuf file)
                     #self.__codeType = 0
                    # self.__nCode = None
                    # self.__nBaud = None
                     self.__code = self.code
                     self.__codeType = 0
                     if self.code != None:
                         self.__codeType = 1
                     self.__nCode = self.nCode
                     self.__nBaud = self.nBaud
                     #self.__code = 0
                     #filling system header parameters
                     self.__nSamples = self.nsa
                     self.newProfiles = self.nprofiles/self.nchannels
                     self.__channelList = list(range(self.nchannels))
                     self.__frequency = self.frequency[0][0]
+                    return 1
                 def createBuffers(self):
                     pass
                 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print('Path:%s does not exists'%self.path)
                     return
                 def __selDates(self, amisr_dirname_format):
                     try:
                         year = int(amisr_dirname_format[0:4])
                         month = int(amisr_dirname_format[4:6])
                         dom = int(amisr_dirname_format[6:8])
                         thisDate = datetime.date(year,month,dom)
                         if (thisDate>=self.startDate and thisDate <= self.endDate):
                             return amisr_dirname_format
                     except:
                         return None
                 def __findDataForDates(self,online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                     dirnameList = [x for x in dirnameList if x!=None]
                     dirnameList = [x.string for x in dirnameList]
                     if not(online):
                         dirnameList = [self.__selDates(x) for x in dirnameList]
                         dirnameList = [x for x in dirnameList if x!=None]
                     if len(dirnameList)>0:
                         self.status = 1
                         self.dirnameList = dirnameList
                         self.dirnameList.sort()
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                     print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
                     print('........................................')
                     filter_filenameList = []
                     self.filenameList.sort()
                     #for i in range(len(self.filenameList)-1):
                     for i in range(len(self.filenameList)):
                         filename = self.filenameList[i]
                         fp = h5py.File(filename,'r')
                         time_str = fp.get('Time/RadacTimeString')
                         startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
                         #startDateTimeStr_File = "2019-12-16 09:21:11"
                         junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         #endDateTimeStr_File = "2019-12-16 11:10:11"
                         endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
                         junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         fp.close()
                         #print("check time", startDateTime_File)
                         if self.timezone == 'lt':
                             startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                             endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
                         if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
                             #self.filenameList.remove(filename)
                             filter_filenameList.append(filename)
                         if (endDateTime_File>=endDateTime_Reader):
                             break
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __filterByGlob1(self, dirName):
                     filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
                     filter_files.sort()
                     filterDict = {}
                     filterDict.setdefault(dirName)
                     filterDict[dirName] = filter_files
                     return filterDict
                 def __getFilenameList(self, fileListInKeys, dirList):
                     for value in fileListInKeys:
                         dirName = list(value.keys())[0]
                         for file in value[dirName]:
                             filename = os.path.join(dirName, file)
                             self.filenameList.append(filename)
                 def __selectDataForTimes(self, online=False):
                     #aun no esta implementado el filtro for tiempo
                     if not(self.status):
                         return None
                     dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                     fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
                     self.__getFilenameList(fileListInKeys, dirList)
                     if not(online):
                         #filtro por tiempo
                         if not(self.all):
                             self.__getTimeFromData()
                         if len(self.filenameList)>0:
                             self.status = 1
                             self.filenameList.sort()
                         else:
                             self.status = 0
                             return None
                     else:
                         #get the last file - 1
                         self.filenameList = [self.filenameList[-2]]
                     new_dirnameList = []
                     for dirname in self.dirnameList:
                         junk = numpy.array([dirname in x for x in self.filenameList])
                         junk_sum = junk.sum()
                         if junk_sum > 0:
                             new_dirnameList.append(dirname)
                     self.dirnameList = new_dirnameList
                     return 1
                 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),walk=True):
                     if endDate ==None:
                      startDate = datetime.datetime.utcnow().date()
                      endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
                     self.__checkPath()
                     self.__findDataForDates(online=True)
                     self.dirnameList = [self.dirnameList[-1]]
                     self.__selectDataForTimes(online=True)
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print("%s" %(self.filenameList[i]))
                     return
                 def __setNextFileOffline(self):
-                    idFile = self.fileIndex
-                    while (True):
-                        idFile += 1
-                        if not(idFile < len(self.filenameList)):
-                            self.flagNoMoreFiles = 1
-                            print("No more Files")
-                            return 0
-                        filename = self.filenameList[idFile]
-                        amisrFilePointer = h5py.File(filename,'r')
+                    try:
+                        self.filename = self.filenameList[self.fileIndex]
-                        break
+                        self.amisrFilePointer = h5py.File(self.filename,'r')
+                        self.fileIndex += 1
+                    except:
+                        self.flagNoMoreFiles = 1
+                        print("No more Files")
+                        return 0
                     self.flagIsNewFile = 1
-                    self.fileIndex = idFile
-                    self.filename = filename
-                    self.amisrFilePointer = amisrFilePointer
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setNextFileOnline(self):
                     filename = self.filenameList[0]
                     if self.__filename_online != None:
                         self.__selectDataForTimes(online=True)
                         filename = self.filenameList[0]
                         wait = 0
                         #self.__waitForNewFile=5 ## DEBUG:
                         while self.__filename_online == filename:
                             print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
                             if wait == 5:
                                 self.flagNoMoreFiles = 1
                                 return 0
                             sleep(self.__waitForNewFile)
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                             wait += 1
                     self.__filename_online = filename
                     self.amisrFilePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def readData(self):
                     buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
                     re = buffer[:,:,:,0]
                     im = buffer[:,:,:,1]
                     dataset = re + im*1j
                     self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
                     timeset = self.radacTime[:,0]
                     return dataset,timeset
                 def reshapeData(self):
                 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
                     channels = self.beamCodeByPulse[0,:]
                     nchan = self.nchannels
                     #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
                     nblocks = self.nblocks
                     nsamples = self.nsa
                     #Dimensions : nChannels, nProfiles, nSamples
                     new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
                     ############################################
                     for thisChannel in range(nchan):
                         new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
                     new_block = numpy.transpose(new_block, (1,0,2,3))
                     new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
                     return new_block
                 def updateIndexes(self):
                     pass
                 def fillJROHeader(self):
                     #fill radar controller header
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
                                                                                   txA=self.__txA,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeight,
                                                                                   deltaHeight=self.__deltaHeight,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code,
                                                                                   fClock=1)
                     #fill system header
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=self.newProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14,
                                                                 pciDioBusWidth=32)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = self.newProfiles*self.nblocks
                     #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     ranges = numpy.reshape(self.rangeFromFile.value,(-1))
                     self.dataOut.heightList =  ranges/1000.0 #km
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     #self.dataOut.timeZone = -5 #self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     if self.timezone == 'lt':
                         self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
                     else:
                         self.dataOut.timeZone = 0 #by default time is UTC
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = self.ippSeconds
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.online
                     pass
                 def readNextFile(self,online=False):
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         self.dataOut.error = True
                         return 0
-                    #if self.__firstFile:
-                    self.readAMISRHeader(self.amisrFilePointer)
-                    self.createBuffers()
+                    if not self.readAMISRHeader(self.amisrFilePointer):
+                        self.dataOut.error = True
+                        return 0
+                    self.createBuffers()
                     self.fillJROHeader()
                     #self.__firstFile = False
                     self.dataset,self.timeset = self.readData()
                     if self.endDate!=None:
                      endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                      time_str = self.amisrFilePointer.get('Time/RadacTimeString')
                      startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
                      junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                      startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                      if self.timezone == 'lt':
                       startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                      if (startDateTime_File>endDateTime_Reader):
                          return 0
                     self.jrodataset = self.reshapeData()
                     #----self.updateIndexes()
                     self.profileIndex = 0
                     return 1
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= (self.newProfiles*self.nblocks):
                         return 1
                     return 0
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     if self.__hasNotDataInBuffer():
                         if not (self.readNextFile(self.online)):
                             return 0
                     if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
                         self.dataOut.flagNoData = True
                         return 0
                     #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
                     self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
                     #print("R_t",self.timeset)
                     #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
                     #verificar basic header de jro data y ver si es compatible con este valor
                     #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
                     indexprof = numpy.mod(self.profileIndex, self.newProfiles)
                     indexblock = self.profileIndex/self.newProfiles
                     #print (indexblock, indexprof)
                     diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
                     diffUTC = 0
                     t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
                     #print("utc :",indexblock," __ ",t_comp)
                     #print(numpy.shape(self.timeset))
                     self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
                     #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
                     #print(self.dataOut.utctime)
                     self.dataOut.profileIndex = self.profileIndex
                     self.dataOut.flagNoData = False
             #         if indexprof == 0:
             #             print self.dataOut.utctime
                     self.profileIndex += 1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     #print("running kamisr")
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

schainpy/scripts/amisr_eej_proc_offline_v3.py +12 -12

             import os, sys
             import time
             import datetime
             path = os.path.dirname(os.getcwd())
             path = os.path.dirname(path)
             sys.path.insert(0, path)
             from schainpy.controller import Project
             def main():
                 desc = "AMISR EEJ Experiment"
                 filename = "amisr_reader.xml"
                 xmin = '07'
                 xmax = '18' #-> el plot genera +1 en la hora, es decir aparece 18 como máximo
                 ymin = '0'
                 ymax = '300'
                 dbmin = '45' #'60'#'55' #'40' #noise  esf  eej
                 dbmax = '65' #'70' #'55'
                 showSPC = '1' #view plot Spectra
                 showRTI = '1' #view plot RTI
                 showNOISE = '1' #view plot NOISE
                 localtime='1' #para ajustar el horario en las gráficas '0' para dejar en utc
                 code = '1,-1,-1,-1,1,1,1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,1,-1,1,1,-1,1'
                 nCode = '1'
                 nBaud = '28'
                 nosamp = '1' # oversample for EEJ
                 today = time.strftime("%Y/%m/%d")
                 #startDate=today
                 #endDate=today
                 startDate='2021/07/11'
                 endDate='2021/07/11'
                 #inPath= '/home/soporte/dataAMISR_test/'
                 inPath= '/home/soporte/dataAMISR/'
                 inPath= '/media/soporte/UARS_4T_D02/AMISR_DATA/2021/'
-                #inPath = '/mnt/data_amisr'
+                #inPath = '/home/soporte/'
                 outPath = '/home/soporte/Data/EEJ'
                 ##.......................................................................................
                 ##.......................................................................................
                 #l = startDate.split('/')                        #adding day of the year to outPath
                 l = startDate.split('/')
                 datelist = datetime.date(int(l[0]),int(l[1]),int(l[2]))
                 DOY = datelist.timetuple().tm_yday
                 outPath= outPath+"/EEJ"+l[0]+str(DOY)
                 if os.path.exists(outPath):
                     print("outPath", outPath)
                 else :
                     os.mkdir(outPath)
                     print("Creating...", outPath)
                 ##.......................................................................................
                 ##.......................................................................................
                 controllerObj = Project()
                 controllerObj.setup(id = '11', name='eej_proc', description=desc)
                 ##.......................................................................................
                 ##.......................................................................................
                 readUnitConfObj = controllerObj.addReadUnit(datatype='AMISRReader',
                                                             path=inPath,
                                                             startDate=startDate,#startDate, #'2014/10/07',
                                                             endDate=endDate, #endDate '2014/10/07',
-                                                            startTime='07:01:30',#'07:00:00',
+                                                            startTime='17:55:30',#'07:00:00',
                                                             endTime='19:00:00',#'15:00:00',
                                                             walk=1,
                                                             code = code,
                                                             nCode = nCode,
                                                             nBaud = nBaud,
                                                             timezone='lt',
                                                             online=0)
-                #AMISR Processing Unit
+                # #AMISR Processing Unit
-                ##.......................................................................................
+                # ##.......................................................................................
-                ##.......................................................................................
+                # ##.......................................................................................
                 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
                 opObj10 = procUnitConfObj0.addOperation(name='setAttribute')
                 opObj10.addParameter(name='frequency', value='445.09e6')
                 # opObj10 = procUnitConfObj0.addOperation(name='setRadarFrequency')
                 # opObj10.addParameter(name='frequency', value='445e6', format='float')
                 opObj01 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
                 opObj01.addParameter(name='code', value=code, format='floatlist')
                 opObj01.addParameter(name='nCode', value=nCode, format='int')
                 opObj01.addParameter(name='nBaud', value=nBaud, format='int')
                 opObj01.addParameter(name='osamp', value=nosamp, format='int')
                 # opObj02 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
                 # opObj02.addParameter(name='n', value='2', format='int')
+                #
+                #
-                ##.......................................................................................
+                # ##.......................................................................................
-                ##.......................................................................................
+                # ##.......................................................................................
                 procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId())
                 procUnitConfObj1.addParameter(name='nFFTPoints', value='16', format='int')
                 opObj11 = procUnitConfObj1.addOperation(name='IncohInt', optype='other')
                 opObj11.addParameter(name='n', value='150', format='int') #300?
                 ## Remove DC signal
                 opObj11 = procUnitConfObj1.addOperation(name='removeDC')
                 ##.......................................................................................
                 ##.......................................................................................
                 # opObj13 = procUnitConfObj1.addOperation(name='getNoise' , optype ='self')
                 # opObj13.addParameter(name='minHei', value='100', format='float')
                 # opObj13.addParameter(name='maxHei', value='280', format='float')
                 #
                 opObj12 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
                 opObj12.addParameter(name='id', value='21', format='int')
                 opObj12.addParameter(name= 'xaxis', value='velocity')
                 opObj12.addParameter(name='ymax', value=ymax, format='int')
                 opObj12.addParameter(name='showprofile', value='1', format='int')
                 opObj12.addParameter(name='wintitle', value='AMISR Beam 0', format='str')
                 opObj12.addParameter(name='zmin', value=dbmin, format='int')
                 opObj12.addParameter(name='zmax', value=dbmax, format='int')
                 opObj12.addParameter(name='save', value=outPath+'/plots', format='str')
                 opObj12.addParameter(name='colormap', value='jet', format='str')
                 opObj12.addParameter(name='localtime', value=localtime,format='int')
                 opObj12.addParameter(name='show', value = showSPC, format='int')
                 ##Generate *.pdata from AMISR data
                 ##.......................................................................................
                 ##.......................................................................................
                 opObj13 = procUnitConfObj1.addOperation(name='SpectraWriter', optype='external')
                 opObj13.addParameter(name='path', value=outPath)
                 opObj13.addParameter(name='blocksPerFile', value='10', format='int')
                 opObj14 = procUnitConfObj1.addOperation(name='NoisePlot', optype='external')
                 opObj14.addParameter(name='id', value='3', format='int')
                 opObj14.addParameter(name='wintitle', value='title0', format='str')
                 opObj14.addParameter(name='showprofile', value='0', format='int')
                 opObj14.addParameter(name='tmin', value=xmin, format='int')
                 opObj14.addParameter(name='tmax', value=xmax, format='int')
                 opObj14.addParameter(name='ymin', value=dbmin, format='int')
                 opObj14.addParameter(name='ymax', value=dbmax, format='int')
                 opObj14.addParameter(name='save', value=outPath, format='str')
                 opObj14.addParameter(name='localtime', value=localtime,format='int')
                 opObj14.addParameter(name='show', value = showNOISE, format='int')
+                # #
                 opObj15 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
                 opObj15.addParameter(name='id', value='2', format='int')
                 opObj15.addParameter(name='localtime', value=localtime,format='int')
                 opObj15.addParameter(name='wintitle', value='RTI', format='str')
                 opObj15.addParameter(name='tmin', value=xmin, format='int')
                 opObj15.addParameter(name='tmax', value=xmax, format='int') #max value =23
                 opObj15.addParameter(name='ymin', value=ymin, format='int')
                 opObj15.addParameter(name='zmin', value=dbmin, format='int')
                 opObj15.addParameter(name='zmax', value=dbmax, format='int')
                 opObj15.addParameter(name='showprofile', value='0', format='int')
                 opObj15.addParameter(name='save', value=outPath+'/plots', format='str')
                 opObj15.addParameter(name='colormap', value='jet', format='str')
                 opObj15.addParameter(name='show', value = showRTI, format='int')
                 ##.......................................................................................
                 ##.......................................................................................
                 procUnitConfObj2 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj1.getId())
                 opObj16 =  procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
                 #Using HDFWriter::::
-                ##.......................................................................................
+                #.......................................................................................
-                ##.......................................................................................
+                #.......................................................................................
                 opObj17 = procUnitConfObj2.addOperation(name='HDFWriter', optype='external')
                 opObj17.addParameter(name='path', value=outPath)
                 opObj17.addParameter(name='blocksPerFile', value='10', format='int')
                 opObj17.addParameter(name='metadataList',value='type,inputUnit,heightList',format='list')
                 opObj17.addParameter(name='dataList',value='moments,data_SNR,utctime',format='list')
                 ##.......................................................................................
                 ##.......................................................................................
                 #print("Escribiendo el archivo XML",controllerObj.writeXml(outPath +'/'+filename))
                 controllerObj.start()
                 #print("Leyendo el archivo XML",controllerObj.readXml(outPath +'/'+filename))
                 ##.......................................................................................
                 ##.......................................................................................
             if __name__ == '__main__':
                 import time
                 start_time = time.time()
                 main()
                 print("--- %s seconds ---" % (time.time() - start_time))

schainpy/scripts/test1.py +50 -76

             import os, sys
             import time
             import datetime
             path = os.path.dirname(os.getcwd())
             path = os.path.dirname(path)
             sys.path.insert(0, path)
             from schainpy.controller import Project
             def main():
                 desc = "AMISR Experiment"
                 filename = "amisr_reader.xml"
                 controllerObj = Project()
                 controllerObj.setup(id = '10', name='eej_proc', description=desc)
                 #path = os.path.join(os.environ['HOME'],'amisr')
                 # path = '/media/signalchain/HD-PXU2/AMISR_JULIA_MODE'
                 # path = '/media/soporte/E9F4-F053/AMISR/Data/NoiseTest/EEJ'
                 # path = '/media/soporte/E9F4-F053/AMISR/Data/ESF'
                 #path = '/mnt/data_amisr'
-                #path = '/media/soporte/AMISR_104'
+                path = '/home/soporte/Data/EEJ/EEJ2021192'
                 #figpath = os.path.join(os.environ['HOME'],'Pictures/amisr/test/proc/esf')
                 #figpath = '/media/soporte/E9F4-F053/AMISR/Data/JULIA/ESF'
-                figpath = '/home/soporte/Data/EEJ'
+                figpath = '/home/soporte/Data/OutTest/EEJ'
                 xmin = '07'
-                xmax = '17' #-> el plot genera +1 en la hora, es decir aparece 18 como máximo
+                xmax = '18' #-> el plot genera +1 en la hora, es decir aparece 18 como máximo
                 ymin = '0'
                 ymax = '300'
                 dbmin = '45' #'60'#'55' #'40' #noise  esf  eej
                 dbmax = '65' #'70' #'55'
                 showSPC = '1' #view plot Spectra
                 showRTI = '1' #view plot RTI
                 showNOISE = '0' #view plot NOISE
                 localtime='1' #para ajustar el horario en las gráficas '0' para dejar en utc
                 code = '1,-1,-1,-1,1,1,1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,1,-1,1,1,-1,1'
                 nCode = '1'
                 nBaud = '28'
                 nosamp = '1' # oversample for EEJ
                 str0 = datetime.date.today()
                 str1 = str0 + datetime.timedelta(days=1)
                 str2 = str0 - datetime.timedelta(days=1)
                 today = str0.strftime("%Y/%m/%d")
                 tomorrow = str1.strftime("%Y/%m/%d")
                 yesterday = str2.strftime("%Y/%m/%d")
                 #print(today,yesterday,tomorrow)
                 #path= '/home/soporte/dataAMISR_test/'
-                path= '/home/soporte/dataAMISR/'
                 #path= '/home/soporte/dataAMISR/'
                 #path='/home/soporte/Documentos/' #
-                readUnitConfObj = controllerObj.addReadUnit(datatype='AMISRReader',
+                readUnitConfObj = controllerObj.addReadUnit(datatype='Spectra',
                                                             path=path,
-                                                            startDate="2019/12/16",#yesterday, #'2014/10/07',
+                                                            startDate='2021/07/11',
-                                                            endDate="2019/12/16", #'2014/10/07',
+                                                            endDate='2021/07/11',
-                                                            startTime='08:00:00',#'07:00:00',
+                                                            startTime='07:01:30',#'07:00:00',
-                                                            endTime='08:58:00',#'15:00:00',
+                                                            endTime='18:00:00',#'15:00:00',
-                                                            walk=0,
+                                                            walk=1,
-                                                            code = code,
-                                                            nCode = nCode,
-                                                            nBaud = nBaud,
                                                             timezone='lt',
                                                             online=0)
                 #AMISR Processing Unit
-                procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
-                opObj10 = procUnitConfObj0.addOperation(name='setRadarFrequency')
-                opObj10.addParameter(name='frequency', value='445e6', format='float')
-                opObj01 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
+                proc_spectra = controllerObj.addProcUnit(datatype='SpectraProc', inputId=readUnitConfObj.getId())
-                opObj01.addParameter(name='code', value=code, format='floatlist')
+                proc_spectra.addParameter(name='nFFTPoints', value='16', format='int')
-                opObj01.addParameter(name='nCode', value=nCode, format='int')
-                opObj01.addParameter(name='nBaud', value=nBaud, format='int')
-                opObj01.addParameter(name='osamp', value=nosamp, format='int')
-                # opObj02 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
-                # opObj02.addParameter(name='n', value='5', format='int')
-                procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraProc', inputId=procUnitConfObj0.getId())
-                procUnitConfObj1.addParameter(name='nFFTPoints', value='16', format='int')
                 #procUnitConfObj1.addParameter(name='frequency', value='445e6', format='float') #no lo acepta
-                opObj11 = procUnitConfObj1.addOperation(name='IncohInt', optype='other')
+                # op1 = proc_spectra.addOperation(name='IncohInt', optype='other')
-                opObj11.addParameter(name='n', value='150', format='int') #300 normal value
+                # op1.addParameter(name='n', value='150', format='int') #300 normal value
-                opObj11 = procUnitConfObj1.addOperation(name='removeDC')
+                #op2 = proc_spectra.addOperation(name='removeDC')
+                op2 = proc_spectra.addOperation(name='CrossSpectraPlot', optype='external')
+                op2.addParameter(name='id', value='10', format='int')
+                op2.addParameter(name='save', value=figpath, format='str')
+                #op2.addParameter(name='zmin', value='10.0', format='float')
+                #op2.addParameter(name='zmax', value='35.0', format='float')
                 # #
-                # opObj12 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
-                # opObj12.addParameter(name='id', value='21', format='int')
+                op3 = proc_spectra.addOperation(name='SpectraPlot', optype='external')
-                # opObj12.addParameter(name='xaxis', value='velocity')
+                op3.addParameter(name='id', value='21', format='int')
-                # opObj12.addParameter(name='ymax', value=ymax, format='int')
+                op3.addParameter(name='xaxis', value='velocity')
-                # opObj12.addParameter(name='showprofile', value='1', format='int')
+                op3.addParameter(name='ymax', value=ymax, format='int')
-                # opObj12.addParameter(name='wintitle', value='AMISR Beam 0', format='str')
+                op3.addParameter(name='showprofile', value='1', format='int')
-                # opObj12.addParameter(name='zmin', value=dbmin, format='int')
+                op3.addParameter(name='wintitle', value='AMISR Beam 0', format='str')
-                # opObj12.addParameter(name='zmax', value=dbmax, format='int')
+                op3.addParameter(name='zmin', value=dbmin, format='int')
-                # opObj12.addParameter(name='save', value=figpath, format='str')
+                op3.addParameter(name='zmax', value=dbmax, format='int')
-                # opObj12.addParameter(name='colormap', value='jet', format='str')
+                op3.addParameter(name='save', value=figpath, format='str')
-                # opObj12.addParameter(name='localtime', value=localtime,format='int')
+                op3.addParameter(name='colormap', value='jet', format='str')
-                # opObj12.addParameter(name='show', value = showSPC, format='int')
+                op3.addParameter(name='localtime', value=localtime,format='int')
+                op3.addParameter(name='show', value = showSPC, format='int')
-                opObj13 = procUnitConfObj1.addOperation(name='getNoise' , optype ='self')
-                opObj13.addParameter(name='minHei', value='100', format='float')
-                opObj13.addParameter(name='maxHei', value='280', format='float')
                 #
-                # ##Generate *.pdata from AMISR data
-                # opObj13 = procUnitConfObj1.addOperation(name='SpectraWriter', optype='external')
-                # opObj13.addParameter(name='path', value=figpath+'SpectraPlot')
-                # opObj13.addParameter(name='blocksPerFile', value='10', format='int')
-                # # opObj11.addParameter(name='datatype', value="4", format="int") #no incluir,x revisar
                 # opObj31 = procUnitConfObj1.addOperation(name='selectChannels')
                 # opObj31.addParameter(name='channelList', value='6,7,8,9', format='intlist')
                 #
-                opObj15 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
+                op4 = proc_spectra.addOperation(name='RTIPlot', optype='external')
-                opObj15.addParameter(name='id', value='2', format='int')
+                op4.addParameter(name='id', value='2', format='int')
-                opObj15.addParameter(name='localtime', value=localtime,format='int')
+                op4.addParameter(name='localtime', value=localtime,format='int')
-                opObj15.addParameter(name='wintitle', value='RTI', format='str')
+                op4.addParameter(name='wintitle', value='RTI', format='str')
-                #opObj15.addParameter(name='xmin', value=xmin, format='int')
+                #op4.addParameter(name='xmin', value=xmin, format='int')
-                opObj15.addParameter(name='xmax', value=xmax, format='int') #max value =23
+                op4.addParameter(name='xmax', value=xmax, format='int') #max value =23
-                opObj15.addParameter(name='ymin', value=ymin, format='int')
+                op4.addParameter(name='ymin', value=ymin, format='int')
-                #opObj15.addParameter(name='ymax', value=ymax, format='int')
+                #op4.addParameter(name='ymax', value=ymax, format='int')
-                opObj15.addParameter(name='zmin', value=dbmin, format='int')
+                op4.addParameter(name='zmin', value=dbmin, format='int')
-                opObj15.addParameter(name='zmax', value=dbmax, format='int')
+                op4.addParameter(name='zmax', value=dbmax, format='int')
-                opObj15.addParameter(name='showprofile', value='1', format='int')
+                op4.addParameter(name='showprofile', value='1', format='int')
-                opObj15.addParameter(name='show', value = showRTI, format='int') #
+                op4.addParameter(name='show', value = showRTI, format='int') #
-                #opObj15.addParameter(name='timerange', value=str(24*60*60), format='int')#conflicto datetime
+                #op4.addParameter(name='timerange', value=str(24*60*60), format='int')#conflicto datetime
-                opObj15.addParameter(name='save', value=figpath+'/plots', format='str')
+                op4.addParameter(name='save', value=figpath+'/plots', format='str')
-                #opObj15.addParameter(name='figpath', value = figpath+'/plots', format='str')#no es necesario
+                #op4.addParameter(name='figpath', value = figpath+'/plots', format='str')#no es necesario
-                opObj15.addParameter(name='colormap', value='jet', format='str')
+                op4.addParameter(name='colormap', value='jet', format='str')
-                opObj15.addParameter(name='channels', value='2,4', format='intlist')
+                op4.addParameter(name='channels', value='2,4', format='intlist')
                 # opObj14 = procUnitConfObj1.addOperation(name='NoisePlot', optype='external')
                 # opObj14.addParameter(name='id', value='3', format='int')
                 # opObj14.addParameter(name='wintitle', value='title0', format='str')
                 # opObj14.addParameter(name='showprofile', value='0', format='int')
                 # opObj14.addParameter(name='xmin', value=xmin, format='int')
                 # opObj14.addParameter(name='xmax', value=xmax, format='int')
                 # opObj14.addParameter(name='ymin', value=dbmin, format='int')
                 # opObj14.addParameter(name='ymax', value=dbmax, format='int')
                 # opObj14.addParameter(name='save', value=figpath+'/plots', format='str')
                 # opObj14.addParameter(name='localtime', value=localtime,format='int')
                 # opObj14.addParameter(name='show', value = showNOISE, format='int')
                 # #
                 # opObj18 = procUnitConfObj1.addOperation(name='PublishData', optype='other')
                 # opObj18.addParameter(name='zeromq', value=1, format='int')
                 # opObj18.addParameter(name='server', value='tcp://0.0.0.0:8020/', format='str')
                 # opObj18.addParameter(name='delay', value=0, format='int')
                 # #
                 # patternX = 'local, remote, ext, period, exp_code, sub_exp_code'
                 #
                 # opObj18 = procUnitConfObj1.addOperation(name='SendToFTP', optype='external')
                 # opObj18.addParameter(name='server', value='localhost', format='str')
                 # opObj18.addParameter(name='username', value='soporte', format='str')
                 # opObj18.addParameter(name='password', value='soporte', format='str')
                 # opObj18.addParameter(name='timeout', value=5, format='int')
                 #
                 # #opObj18.addParameter(name='patternx', value=patternX, format='str')
                 # opObj18.addParameter(name='patternlocalfolder', value=figpath, format='str')
                 # opObj18.addParameter(name='patternremotefolder', value=remotefolder, format='str')
                 # opObj18.addParameter(name='patternext', value='.png', format='str')
                 # opObj18.addParameter(name='patternperiod', value=1, format='int')
                 # opObj18.addParameter(name='patternprotocol', value='ftp', format='str')
                 # opObj18.addParameter(name='patternsub_ext', value='amisr', format='str')
                 # remotefolder = '/Data/myServer/'
                 # opObj18 = controllerObj.addProcUnit(name='SendToServer', inputId=procUnitConfObj1.getId())
                 # opObj18.addParameter(name='server', value='localhost', format='str')
                 # opObj18.addParameter(name='username', value='soporte', format='str')
                 # opObj18.addParameter(name='password', value='soporte', format='str')
                 # opObj18.addParameter(name='localfolder', value=figpath, format='str')
                 # opObj18.addParameter(name='remotefolder', value=remotefolder, format='str')
                 # opObj18.addParameter(name='ext', value='.png', format='str')
                 # opObj18.addParameter(name='period', value=5, format='int')
                 # opObj18.addParameter(name='protocol', value='ftp', format='str')
                 # #-----------------------------------------------------------------------------------------------
                 #####################
                 #
                 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=procUnitConfObj1.getId())
                 # opObj16 =  procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
                 #
                 #
                 # #Using ParamWriter::::
                 # opObj17 = procUnitConfObj2.addOperation(name='ParamWriter', optype='external')
                 # opObj17.addParameter(name='path', value=figpath+'/param')
                 # opObj17.addParameter(name='blocksPerFile', value='10', format='int')
                 # opObj17.addParameter(name='metadataList',value='type,inputUnit,heightList',format='list')
                 # opObj17.addParameter(name='dataList',value='moments,data_SNR,utctime',format='list')
                 # opObj17.addParameter(name='mode',value='1',format='int') #'0' channels, '1' parameters, '3' table (for meteors)
                 #
                 #
                 #print("Escribiendo el archivo XML",controllerObj.writeXml(path +'/'+filename))
                 controllerObj.start()
                 #print("Leyendo el archivo XML",controllerObj.readXml(path +'/'+filename))
             if __name__ == '__main__':
                 import time
                 start_time = time.time()
                 main()
                 print("--- %s seconds ---" % (time.time() - start_time))

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages