schain Commit - r1231:a7a69d889908 · Jicamarca Repository

Merge branch 'v3.0-devel' of http://jro-dev.igp.gob.pe/rhodecode/schain into v3.0-devel

Juan C. Espinoza -

r1231:a7a69d889908

parent child

Context file:

r1231:a7a69d889908

Collapse all files

schainpy/model/io/jroIO_param.py +3 -1

             import numpy
             import time
             import os
             import h5py
             import re
             import datetime
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             # from .jroIO_base import *
             from schainpy.model.io.jroIO_base import *
             import schainpy
             from schainpy.utils import log
             @MPDecorator
             class ParamReader(JRODataReader,ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 path
                 startDate
                 endDate
                 startTime
                 endTime
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 timezone = None
                 startTime = None
                 endTime = None
                 fileIndex = None
                 utcList = None      #To select data in the utctime list
                 blockList = None    #List to blocks to be read from the file
                 blocksPerFile = None    #Number of blocks to be read
                 blockIndex = None
                 path = None
                 #List of Files
                 filenameList = None
                 datetimeList = None
                 #Hdf5 File
                 listMetaname = None
                 listMeta = None
                 listDataname = None
                 listData = None
                 listShapes = None
                 fp = None
                 #dataOut reconstruction
                 dataOut = None
                 def __init__(self):#, **kwargs):
                     ProcessingUnit.__init__(self) #, **kwargs)
                     self.dataOut = Parameters()
                     return
                 def setup(self, **kwargs):
                     path = kwargs['path']
                     startDate = kwargs['startDate']
                     endDate = kwargs['endDate']
                     startTime = kwargs['startTime']
                     endTime = kwargs['endTime']
                     walk = kwargs['walk']
                     if 'ext' in kwargs:
                         ext = kwargs['ext']
                     else:
                         ext = '.hdf5'
                     if 'timezone' in kwargs:
                         self.timezone = kwargs['timezone']
                     else:
                         self.timezone = 'lt'
                     print("[Reading] Searching files in offline mode ...")
                     pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            ext=ext, walk=walk)
                     if not(filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.startTime = startTime
                     self.endTime = endTime
                     self.__readMetadata()
                     self.__setNextFileOffline()
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate=None,
                                         endDate=None,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         ext='.hdf5',
                                         walk=True):
                     expLabel = ''
                     self.filenameList = []
                     self.datetimeList = []
                     pathList = []
                     JRODataObj = JRODataReader()
                     dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
                     if dateList == []:
                         print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     if len(dateList) > 1:
                         print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
                     else:
                         print("[Reading] data was found for the date %s" %(dateList[0]))
                     filenameList = []
                     datetimeList = []
                     #----------------------------------------------------------------------------------
                     for thisPath in pathList:
              #             thisPath = pathList[pathDict[file]]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for file in fileList:
                             filename = os.path.join(thisPath,file)
                             if not isFileInDateRange(filename, startDate, endDate):
                                 continue
                             thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
                         return None, None
                     print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
                     print()
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
                     """
                     Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
                     Inputs:
                         filename            :    nombre completo del archivo de datos en formato Jicamarca (.r)
                         startDate          :    fecha inicial del rango seleccionado en formato datetime.date
                         endDate            :    fecha final del rango seleccionado en formato datetime.date
                         startTime          :    tiempo inicial del rango seleccionado en formato datetime.time
                         endTime            :    tiempo final del rango seleccionado en formato datetime.time
                     Return:
                         Boolean    :    Retorna True si el archivo de datos contiene datos en el rango de
                                         fecha especificado, de lo contrario retorna False.
                     Excepciones:
                         Si el archivo no existe o no puede ser abierto
                         Si la cabecera no puede ser leida.
                     """
                     try:
                         fp = h5py.File(filename,'r')
                         grp1 = fp['Data']
                     except IOError:
                         traceback.print_exc()
                         raise IOError("The file %s can't be opened" %(filename))
                     #chino rata
                     #In case has utctime attribute
                     grp2 = grp1['utctime']
              #         thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
                     thisUtcTime = grp2.value[0]
                     fp.close()
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
              #         thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
                     #General case
                     #           o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
                     #-----------o----------------------------o-----------
                     #       startTime                     endTime
                     if endTime >= startTime:
                         thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
                         if numpy.any(thisUtcLog):   #If there is one block between the hours mentioned
                             return thisDatetime
                         return None
                     #If endTime < startTime then endTime belongs to the next day
                     #<<<<<<<<<<<o                            o>>>>>>>>>>>
                     #-----------o----------------------------o-----------
                     #        endTime                    startTime
                     if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
                         return None
                     if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
                         return None
                     return thisDatetime
                 def __setNextFileOffline(self):
                     self.fileIndex += 1
                     idFile = self.fileIndex
                     if not(idFile < len(self.filenameList)):
                         print("No more Files")
                         return 0
                     filename = self.filenameList[idFile]
                     filePointer = h5py.File(filename,'r')
                     self.filename = filename
                     self.fp = filePointer
                     print("Setting the file: %s"%self.filename)
              #         self.__readMetadata()
                     self.__setBlockList()
                     self.__readData()
              #         self.nRecords = self.fp['Data'].attrs['blocksPerFile']
              #         self.nRecords = self.fp['Data'].attrs['nRecords']
                     self.blockIndex = 0
                     return 1
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     fp = self.fp
                     startTime = self.startTime
                     endTime = self.endTime
                     grp = fp['Data']
                     thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
                     #ERROOOOR
                     if self.timezone == 'lt':
                         thisUtcTime -= 5*3600
                     thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     self.pathMeta
                     self.listShapes
                     self.listMetaname
                     self.listMeta
                     '''
              #         grp = self.fp['Data']
              #         pathMeta = os.path.join(self.path, grp.attrs['metadata'])
              #
              #         if pathMeta == self.pathMeta:
              #             return
              #         else:
              #             self.pathMeta = pathMeta
              #
              #         filePointer = h5py.File(self.pathMeta,'r')
              #         groupPointer = filePointer['Metadata']
                     filename = self.filenameList[0]
                     fp = h5py.File(filename,'r')
                     gp = fp['Metadata']
                     listMetaname = []
                     listMetadata = []
                     for item in list(gp.items()):
                         name = item[0]
                         if name=='array dimensions':
                             table = gp[name][:]
                             listShapes = {}
                             for shapes in table:
                                 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
                         else:
                             data = gp[name].value
                             listMetaname.append(name)
                             listMetadata.append(data)
              #                 if name=='type':
              #                     self.__initDataOut(data)
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     fp.close()
                     return
                 def __readData(self):
                     grp = self.fp['Data']
                     listdataname = []
                     listdata = []
                     for item in list(grp.items()):
                         name = item[0]
                         listdataname.append(name)
                         array = self.__setDataArray(grp[name],self.listShapes[name])
                         listdata.append(array)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def __setDataArray(self, dataset, shapes):
                     nDims = shapes[0]
                     nDim2 = shapes[1]      #Dimension 0
                     nDim1 = shapes[2]      #Dimension 1, number of Points or Parameters
                     nDim0 = shapes[3]      #Dimension 2, number of samples or ranges
                     mode = shapes[4]        #Mode of storing
                     blockList = self.blockList
                     blocksPerFile = self.blocksPerFile
                     #Depending on what mode the data was stored
                     if mode == 0:       #Divided in channels
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     if mode == 1:     #Divided in parameter
                         strds = 'table'
                         nDatas = nDim1
                         newShapes = (blocksPerFile,nDim2,nDim0)
                     elif mode==2:       #Concatenated in a table
                         strds = 'table0'
                         arrayData = dataset[strds].value
                         #Selecting part of the dataset
                         utctime = arrayData[:,0]
                         u, indices = numpy.unique(utctime, return_index=True)
                         if blockList.size != indices.size:
                             indMin = indices[blockList[0]]
                             if blockList[1] + 1 >= indices.size:
                                 arrayData = arrayData[indMin:,:]
                             else:
                                 indMax = indices[blockList[1] + 1]
                                 arrayData = arrayData[indMin:indMax,:]
                         return arrayData
                     #    One dimension
                     if nDims == 0:
                         arrayData = dataset.value.astype(numpy.float)[0][blockList]
                     #    Two dimensions
                     elif nDims == 2:
                         arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
                         newShapes = (blocksPerFile,nDim0)
                         nDatas = nDim1
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             arrayData[:,i,:] = data[blockList,:]
                     #    Three dimensions
                     else:
                         arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
                         for i in range(nDatas):
                             data = dataset[strds + str(i)].value
                             for b in range(blockList.size):
                                 arrayData[b,:,i,:] = data[:,:,blockList[b]]
                     return arrayData
                 def __setDataOut(self):
                     listMeta = self.listMeta
                     listMetaname = self.listMetaname
                     listDataname = self.listDataname
                     listData = self.listData
                     listShapes = self.listShapes
                     blockIndex = self.blockIndex
              #         blockList = self.blockList
                     for i in range(len(listMeta)):
                         setattr(self.dataOut,listMetaname[i],listMeta[i])
                     for j in range(len(listData)):
                         nShapes = listShapes[listDataname[j]][0]
                         mode = listShapes[listDataname[j]][4]
                         if nShapes == 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         elif nShapes > 1:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
                         elif mode==0:
                             setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
                         #Mode Meteors
                         elif mode ==2:
                             selectedData = self.__selectDataMode2(listData[j], blockIndex)
                             setattr(self.dataOut, listDataname[j], selectedData)
                     return
                 def __selectDataMode2(self, data, blockIndex):
                     utctime = data[:,0]
                     aux, indices = numpy.unique(utctime, return_inverse=True)
                     selInd = numpy.where(indices == blockIndex)[0]
                     selData = data[selInd,:]
                     return selData
                 def getData(self):
                     if self.blockIndex==self.blocksPerFile:
                          if not( self.__setNextFileOffline() ):
                             self.dataOut.flagNoData = True
                             return 0
                     self.__setDataOut()
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
              #             self.setObjProperties()
                         self.isConfig = True
                     self.getData()
                     return
             @MPDecorator
             class ParamWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 path:             path where the files will be stored
                 blocksPerFile:    number of blocks that will be saved in per HDF5 format file
                 mode:             selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
                 metadataList:     list of attributes that will be stored as metadata
                 dataList:         list of attributes that will be stores as data
                 '''
                 ext = ".hdf5"
                 optchar = "D"
                 metaoptchar = "M"
                 metaFile = None
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 grp = None
                 ds = None
                 firsttime = True
                 #Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 #Data Arrays
                 dataList = None
                 metadataList = None
                 dsList = None   #List of dictionaries with dataset properties
                 tableDim = None
                 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
                 currentDay = None
                 lastTime = None
+                setType = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = dataList
                     self.dataOut = dataOut
                     self.mode = mode
                     if self.mode is not None:
                         self.mode = numpy.zeros(len(self.dataList)) + mode
                     else:
                         self.mode = numpy.ones(len(self.dataList))
                     self.setType = setType
                     arrayDim = numpy.zeros((len(self.dataList),5))
                     #Table dimensions
                     dtype0 = self.dtype
                     tableList = []
                     #Dictionary and list of tables
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         dataAux = getattr(self.dataOut, self.dataList[i])
                         dsDict['variable'] = self.dataList[i]
                         #---------------------    Conditionals    ------------------------
                         #There is no data
                         if dataAux is None:
                             return 0
                         if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                             dsDict['mode'] = 0
                             dsDict['nDim'] = 0
                             arrayDim[i,0] = 0
                             dsList.append(dsDict)
                         #Mode 2: meteors
                         elif self.mode[i] == 2:
                             dsDict['dsName'] = 'table0'
                             dsDict['mode'] = 2      # Mode meteors
                             dsDict['shape'] = dataAux.shape[-1]
                             dsDict['nDim'] = 0
                             dsDict['dsNumber'] = 1
                             arrayDim[i,3] = dataAux.shape[-1]
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             dsList.append(dsDict)
                         #Mode 1
                         else:
                             arrayDim0 = dataAux.shape       #Data dimensions
                             arrayDim[i,0] = len(arrayDim0)  #Number of array dimensions
                             arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             strtable = 'table'
                             dsDict['mode'] = 1      # Mode parameters
                             # Three-dimension arrays
                             if len(arrayDim0) == 3:
                                 arrayDim[i,1:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,2:4]
                                 dsDict['nDim'] = 3
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # Two-dimension arrays
                             elif len(arrayDim0) == 2:
                                 arrayDim[i,2:-1] = numpy.array(arrayDim0)
                                 nTables = int(arrayDim[i,2])
                                 dsDict['dsNumber'] = nTables
                                 dsDict['shape'] = arrayDim[i,3]
                                 dsDict['nDim'] = 2
                                 for j in range(nTables):
                                     dsDict = dsDict.copy()
                                     dsDict['dsName'] = strtable + str(j)
                                     dsList.append(dsDict)
                             # One-dimension arrays
                             elif len(arrayDim0) == 1:
                                 arrayDim[i,3] = arrayDim0[0]
                                 dsDict['shape'] = arrayDim0[0]
                                 dsDict['dsNumber'] = 1
                                 dsDict['dsName'] = strtable + str(0)
                                 dsDict['nDim'] = 1
                                 dsList.append(dsDict)
                         table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
                         tableList.append(table)
                     self.dsList = dsList
                     self.tableDim = numpy.array(tableList, dtype = dtype0)
                     self.blockIndex = 0
                     timeTuple = time.localtime(dataOut.utctime)
                     self.currentDay = timeTuple.tm_yday
                 def putMetadata(self):
                     fp = self.createMetadataFile()
                     self.writeMetadata(fp)
                     fp.close()
                     return
                 def createMetadataFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = ''
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         filesList = sorted( filesList, key=str.lower )
                         if len( filesList ) > 0:
                             filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     self.metaFile = file
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     return fp
                 def writeMetadata(self, fp):
                     grp = fp.create_group("Metadata")
                     grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
                     for i in range(len(self.metadataList)):
                         grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
                     return
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     if self.lastTime is None:
                         self.lastTime = currentTime
                     #Day
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     #Time
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3*60*60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     mode = self.mode
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if os.path.exists(fullpath):
                         filesList = os.listdir( fullpath )
-                        filesList = [k for k in filesList if k.startswith(self.optchar)]
+                        filesList = [k for k in filesList if 'M' in k]
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber( filen[8:11] ):
                                 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1 #inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     else:
                         setFile = timeTuple.tm_hour*60+timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext )
                     filename = os.path.join( path, subfolder, file )
                     #Setting HDF5 File
                     fp = h5py.File(filename,'w')
                     #write metadata
                     self.writeMetadata(fp)
                     #Write data
                     grp = fp.create_group("Data")
                     ds = []
                     data = []
                     dsList = self.dsList
                     i = 0
                     while i < len(dsList):
                         dsInfo = dsList[i]
                         #One-dimension data
                         if dsInfo['mode'] == 0:
                             ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 2:
                             grp0 = grp.create_group(dsInfo['variable'])
                             ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
                             ds.append(ds0)
                             data.append([])
                             i += 1
                             continue
                         elif dsInfo['mode'] == 1:
                             grp0 = grp.create_group(dsInfo['variable'])
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[i]
                                 tableName = dsInfo['dsName']
                                 if dsInfo['nDim'] == 3:
                                     shape = dsInfo['shape'].astype(int)
                                     ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
                                 else:
                                     shape = int(dsInfo['shape'])
                                     ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
                                 ds.append(ds0)
                                 data.append([])
                                 i += 1
                     fp.flush()
                     fp.close()
                     log.log('creating file: {}'.format(filename), 'Writing')
                     self.filename = filename
                     self.ds = ds
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if self.blockIndex == self.blocksPerFile or self.timeFlag():
                         self.setNextFile()
                     self.readBlock()
                     self.setBlock()     #Prepare data to be written
                     self.writeBlock()   #Write data
                     return
                 def readBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     dsList = self.dsList
                     ds = self.ds
                             #Setting HDF5 File
                     fp = h5py.File(self.filename,'r+')
                     grp = fp["Data"]
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         if dsInfo['mode'] == 0:
                             ds0 = grp[dsInfo['variable']]
                             ds[ind] = ds0
                             ind += 1
                         else:
                             grp0 = grp[dsInfo['variable']]
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[ind]
                                 ds0 = grp0[dsInfo['dsName']]
                                 ds[ind] = ds0
                                 ind += 1
                     self.fp = fp
                     self.grp = grp
                     self.ds = ds
                     return
                 def setBlock(self):
                     '''
                     data Array configured
                     self.data
                     '''
                     #Creating Arrays
                     dsList = self.dsList
                     data = self.data
                     ind = 0
                     while ind < len(dsList):
                         dsInfo = dsList[ind]
                         dataAux = getattr(self.dataOut, dsInfo['variable'])
                         mode = dsInfo['mode']
                         nDim = dsInfo['nDim']
                         if mode == 0 or mode == 2 or nDim == 1:
                             data[ind] = dataAux
                             ind += 1
              #             elif nDim == 1:
              #                 data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
              #                 ind += 1
                         elif nDim == 2:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[j,:]
                                 ind += 1
                         elif nDim == 3:
                             for j in range(dsInfo['dsNumber']):
                                 data[ind] = dataAux[:,j,:]
                                 ind += 1
                     self.data = data
                     return
                 def writeBlock(self):
                     '''
                     Saves the block in the HDF5 file
                     '''
                     dsList = self.dsList
                     for i in range(len(self.ds)):
                         dsInfo = dsList[i]
                         nDim = dsInfo['nDim']
                         mode = dsInfo['mode']
                         #    First time
                         if self.firsttime:
                             if type(self.data[i]) == numpy.ndarray:
                                 if nDim == 3:
                                     self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
                                     self.ds[i].resize(self.data[i].shape)
                                 if mode == 2:
                                     self.ds[i].resize(self.data[i].shape)
                             self.ds[i][:] = self.data[i]
                         else:
                         #    From second time
                             #    Meteors!
                             if mode == 2:
                                 dataShape = self.data[i].shape
                                 dsShape = self.ds[i].shape
                                 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
                                 self.ds[i][dsShape[0]:,:] = self.data[i]
                             #    No dimension
                             elif mode == 0:
                                 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
                                 self.ds[i][0,-1] = self.data[i]
                             #    One dimension
                             elif nDim == 1:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
                                 self.ds[i][-1,:] = self.data[i]
                             #    Two dimension
                             elif nDim == 2:
                                 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
                                 self.ds[i][self.blockIndex,:] = self.data[i]
                             #    Three dimensions
                             elif nDim == 3:
                                 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
                                 self.ds[i][:,:,-1] = self.data[i]
                     self.firsttime = False
                     self.blockIndex += 1
                     #Close to save changes
                     self.fp.flush()
                     self.fp.close()
                     return
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
+                    self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList, mode=mode,
                                    setType=setType)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
   No newline at end of file

schainpy/model/io/jroIO_spectra.py +1 -1

             '''
             Created on Jul 2, 2014
             @author: roj-idl71
             '''
             import numpy
             from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jrodata import Spectra
             from schainpy.utils import log
             @MPDecorator
             class SpectraReader(JRODataReader, ProcessingUnit):
                 """
                 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
                 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
                 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
                                     paresCanalesIguales    * alturas * perfiles  (Self Spectra)
                                     paresCanalesDiferentes * alturas * perfiles  (Cross Spectra)
                                     canales * alturas                            (DC Channels)
                 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
                 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
                 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
                 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
                 Example:
                     dpath = "/home/myuser/data"
                     startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
                     endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
                     readerObj = SpectraReader()
                     readerObj.setup(dpath, startTime, endTime)
                     while(True):
                         readerObj.getData()
                         print readerObj.data_spc
                         print readerObj.data_cspc
                         print readerObj.data_dc
                         if readerObj.flagNoMoreFiles:
                             break
                 """
                 pts2read_SelfSpectra = 0
                 pts2read_CrossSpectra = 0
                 pts2read_DCchannels = 0
                 ext = ".pdata"
                 optchar = "P"
                 dataOut = None
                 nRdChannels = None
                 nRdPairs = None
                 rdPairList = []
                 def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase SpectraReader para la lectura de datos de espectros.
                     Inputs:
                         dataOut    :    Objeto de la clase Spectra. Este objeto sera utilizado para
                                           almacenar un perfil de datos cada vez que se haga un requerimiento
                                           (getData). El perfil sera obtenido a partir del buffer de datos,
                                           si el buffer esta vacio se hara un nuevo proceso de lectura de un
                                           bloque de datos.
                                           Si este parametro no es pasado se creara uno internamente.
                     Affected:
                         self.dataOut
                     Return      : None
                     """
                     #Eliminar de la base la herencia
                     ProcessingUnit.__init__(self)#, **kwargs)
             #         self.isConfig = False
                     self.pts2read_SelfSpectra = 0
                     self.pts2read_CrossSpectra = 0
                     self.pts2read_DCchannels = 0
                     self.datablock = None
                     self.utc = None
                     self.ext = ".pdata"
                     self.optchar = "P"
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.online = 0
                     self.fp = None
                     self.idFile = None
                     self.dtype = None
                     self.fileSizeByHeader = None
                     self.filenameList = []
                     self.filename = None
                     self.fileSize = None
                     self.firstHeaderSize = 0
                     self.basicHeaderSize = 24
                     self.pathList = []
                     self.lastUTTime = 0
                     self.maxTimeStep = 30
                     self.flagNoMoreFiles = 0
                     self.set = 0
                     self.path = None
                     self.delay  = 60   #seconds
                     self.nTries = 3  #quantity tries
                     self.nFiles = 3   #number of files for searching
                     self.nReadBlocks = 0
                     self.flagIsNewFile = 1
                     self.__isFirstTimeOnline = 1
             #         self.ippSeconds = 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     self.nTotalBlocks = 0
                     self.blocksize = 0
                     self.dataOut = self.createObjByDefault()
                     self.profileIndex = 1 #Always
                 def createObjByDefault(self):
                     dataObj = Spectra()
                     return dataObj
                 def __hasNotDataInBuffer(self):
                     return 1
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.nRdChannels
                         self.nRdPairs
                         self.pts2read_SelfSpectra
                         self.pts2read_CrossSpectra
                         self.pts2read_DCchannels
                         self.blocksize
                         self.dataOut.nChannels
                         self.dataOut.nPairs
                     Return:
                         None
                     """
                     self.nRdChannels = 0
                     self.nRdPairs = 0
                     self.rdPairList = []
                     for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
                         if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
                             self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
                         else:
                             self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
                             self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
                     pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
                     self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
                     self.blocksize = self.pts2read_SelfSpectra
                     if self.processingHeaderObj.flag_cspc:
                         self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
                         self.blocksize += self.pts2read_CrossSpectra
                     if self.processingHeaderObj.flag_dc:
                         self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
                         self.blocksize += self.pts2read_DCchannels
             #        self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
                 def readBlock(self):
                     """
                     Lee el bloque de datos desde la posicion actual del puntero del archivo
                     (self.fp) y actualiza todos los parametros relacionados al bloque de datos
                     (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
                     es seteado a 0
                     Return: None
                     Variables afectadas:
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Exceptions:
                         Si un bloque leido no es un bloque valido
                     """
                     blockOk_flag = False
                     fpointer = self.fp.tell()
                     spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
                     spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
                         cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
                     if self.processingHeaderObj.flag_dc:
                         dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
                         dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
                     if not self.processingHeaderObj.shif_fft:
                         #desplaza a la derecha en el eje 2 determinadas posiciones
                         shift = int(self.processingHeaderObj.profilesPerBlock/2)
                         spc = numpy.roll( spc, shift , axis=2 )
                         if self.processingHeaderObj.flag_cspc:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             cspc = numpy.roll( cspc, shift, axis=2 )
                     #Dimensions : nChannels, nProfiles, nSamples
                     spc = numpy.transpose( spc, (0,2,1) )
                     self.data_spc = spc
                     if self.processingHeaderObj.flag_cspc:
                         cspc = numpy.transpose( cspc, (0,2,1) )
                         self.data_cspc = cspc['real'] + cspc['imag']*1j
                     else:
                         self.data_cspc = None
                     if self.processingHeaderObj.flag_dc:
                         self.data_dc = dc['real'] + dc['imag']*1j
                     else:
                         self.data_dc = None
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getFirstHeader(self):
                     self.getBasicHeader()
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
             #         self.dataOut.ippSeconds = self.ippSeconds
             #         self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
                     self.dataOut.dtype = self.dtype
             #         self.dataOut.nPairs = self.nPairs
                     self.dataOut.pairsList = self.rdPairList
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
                     xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
                     self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
                     self.dataOut.flagShiftFFT = True    #Data is always shifted
                     self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
                     self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
                 def getData(self):
                     """
                     First method to execute before "RUN" is called.
                     Copia el buffer de lectura a la clase "Spectra",
                     con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
                     lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
                     Return:
 :    Si no hay mas archivos disponibles
 :    Si hizo una buena copia del buffer
                     Affected:
                         self.dataOut
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     """
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         print('Process finished')
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():
                         if not( self.readNextBlock() ):
                             self.dataOut.flagNoData = True
                             return 0
                     #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
                     if self.data_spc is None:
                         self.dataOut.flagNoData = True
                         return 0
                     self.getBasicHeader()
                     self.getFirstHeader()
                     self.dataOut.data_spc = self.data_spc
                     self.dataOut.data_cspc = self.data_cspc
                     self.dataOut.data_dc = self.data_dc
                     self.dataOut.flagNoData = False
                     self.dataOut.realtime = self.online
                     return self.dataOut.data_spc
             @MPDecorator
             class SpectraWriter(JRODataWriter, Operation):
                 """
                 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
                 de los datos siempre se realiza por bloques.
                 """
                 ext = ".pdata"
                 optchar = "P"
                 shape_spc_Buffer = None
                 shape_cspc_Buffer = None
                 shape_dc_Buffer = None
                 data_spc = None
                 data_cspc = None
                 data_dc = None
                 def __init__(self):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Affected:
                         self.dataOut
                         self.basicHeaderObj
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.processingHeaderObj
                     Return: None
                     """
                     Operation.__init__(self)
                     self.nTotalBlocks = 0
                     self.data_spc = None
                     self.data_cspc = None
                     self.data_dc = None
                     self.fp = None
                     self.flagIsNewFile = 1
                     self.nTotalBlocks = 0
                     self.flagIsNewBlock = 0
                     self.setFile = None
                     self.dtype = None
                     self.path = None
                     self.noMoreFiles = 0
                     self.filename = None
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                 def hasAllDataInBuffer(self):
                     return 1
                 def setBlockDimension(self):
                     """
                     Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
                     Affected:
                         self.shape_spc_Buffer
                         self.shape_cspc_Buffer
                         self.shape_dc_Buffer
                     Return: None
                     """
                     self.shape_spc_Buffer = (self.dataOut.nChannels,
                                              self.processingHeaderObj.nHeights,
                                              self.processingHeaderObj.profilesPerBlock)
                     self.shape_cspc_Buffer = (self.dataOut.nPairs,
                                               self.processingHeaderObj.nHeights,
                                               self.processingHeaderObj.profilesPerBlock)
                     self.shape_dc_Buffer = (self.dataOut.nChannels,
                                             self.processingHeaderObj.nHeights)
                 def writeBlock(self):
                     """processingHeaderObj
                     Escribe el buffer en el file designado
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                         self.flagIsNewFile
                         self.flagIsNewBlock
                         self.nTotalBlocks
                         self.nWriteBlocks
                     Return: None
                     """
                     spc = numpy.transpose( self.data_spc, (0,2,1) )
                     if not self.processingHeaderObj.shif_fft:
                         spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                     data = spc.reshape((-1))
                     data = data.astype(self.dtype[0])
                     data.tofile(self.fp)
                     if self.data_cspc is not None:
                         cspc = numpy.transpose( self.data_cspc, (0,2,1) )
-                        #data = numpy.zeros( numpy.shape(cspc), self.dtype )
+                        data = numpy.zeros( numpy.shape(cspc), self.dtype )
                         #print 'data.shape', self.shape_cspc_Buffer
                         if not self.processingHeaderObj.shif_fft:
                             cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
                         data['real'] = cspc.real
                         data['imag'] = cspc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
                     if self.data_dc is not None:
                         dc = self.data_dc
                         data = numpy.zeros( numpy.shape(dc), self.dtype )
                         data['real'] = dc.real
                         data['imag'] = dc.imag
                         data = data.reshape((-1))
                         data.tofile(self.fp)
             #         self.data_spc.fill(0)
             #
             #         if self.data_dc is not None:
             #             self.data_dc.fill(0)
             #
             #         if self.data_cspc is not None:
             #             self.data_cspc.fill(0)
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nWriteBlocks += 1
                     self.blockIndex += 1
             #         print "[Writing] Block = %d04" %self.blockIndex
                 def putData(self):
                     """
                     Setea un bloque de datos y luego los escribe en un file
                     Affected:
                         self.data_spc
                         self.data_cspc
                         self.data_dc
                     Return:
 :    Si no hay data o no hay mas files que puedan escribirse
 :    Si se escribio la data de un bloque en un file
                     """
                     if self.dataOut.flagNoData:
                         return 0
                     self.flagIsNewBlock = 0
                     if self.dataOut.flagDiscontinuousBlock:
                         self.data_spc.fill(0)
                         if self.dataOut.data_cspc is not None:
                             self.data_cspc.fill(0)
                         if self.dataOut.data_dc is not None:
                             self.data_dc.fill(0)
                         self.setNextFile()
                     if self.flagIsNewFile == 0:
                         self.setBasicHeader()
                     self.data_spc = self.dataOut.data_spc.copy()
                     if self.dataOut.data_cspc is not None:
                         self.data_cspc = self.dataOut.data_cspc.copy()
                     if self.dataOut.data_dc is not None:
                         self.data_dc = self.dataOut.data_dc.copy()
                     # #self.processingHeaderObj.dataBlocksPerFile)
                     if self.hasAllDataInBuffer():
             #            self.setFirstHeader()
                         self.writeNextBlock()
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
                     '''
                     dtype_width = self.getDtypeWidth()
                     pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
                     pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
                     blocksize = (pts2write_SelfSpectra*dtype_width)
                     if self.dataOut.data_cspc is not None:
                         pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
                         blocksize += (pts2write_CrossSpectra*dtype_width*2)
                     if self.dataOut.data_dc is not None:
                         pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
                         blocksize += (pts2write_DCchannels*dtype_width*2)
             #         blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
                     return blocksize
                 def setFirstHeader(self):
                     """
                     Obtiene una copia del First Header
                     Affected:
                         self.systemHeaderObj
                         self.radarControllerHeaderObj
                         self.dtype
                     Return:
                         None
                     """
                     self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
                     self.systemHeaderObj.nChannels = self.dataOut.nChannels
                     self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
                     self.processingHeaderObj.dtype = 1 # Spectra
                     self.processingHeaderObj.blockSize = self.__getBlockSize()
                     self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
                     self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
                     self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
                     self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
                     self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
                     self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
                     self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
                     if self.processingHeaderObj.totalSpectra > 0:
                         channelList = []
                         for channel in range(self.dataOut.nChannels):
                             channelList.append(channel)
                             channelList.append(channel)
                         pairsList = []
                         if self.dataOut.nPairs > 0:
                             for pair in self.dataOut.pairsList:
                                 pairsList.append(pair[0])
                                 pairsList.append(pair[1])
                         spectraComb = channelList + pairsList
                         spectraComb = numpy.array(spectraComb, dtype="u1")
                         self.processingHeaderObj.spectraComb = spectraComb
                     if self.dataOut.code is not None:
                         self.processingHeaderObj.code = self.dataOut.code
                         self.processingHeaderObj.nCode = self.dataOut.nCode
                         self.processingHeaderObj.nBaud = self.dataOut.nBaud
                     if self.processingHeaderObj.nWindows != 0:
                         self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
                         self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                         self.processingHeaderObj.nHeights = self.dataOut.nHeights
                         self.processingHeaderObj.samplesWin = self.dataOut.nHeights
                     self.processingHeaderObj.processFlags = self.getProcessFlags()
                     self.setBasicHeader()

schainpy/model/proc/jroproc_spectra.py +4 -8

             import itertools
             import numpy
             from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
             from schainpy.model.data.jrodata import Spectra
             from schainpy.model.data.jrodata import hildebrand_sekhon
             from schainpy.utils import log
             @MPDecorator
             class SpectraProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.buffer = None
                     self.firstdatatime = None
                     self.profIndex = 0
                     self.dataOut = Spectra()
                     self.id_min = None
                     self.id_max = None
                     self.setupReq = False #Agregar a todas las unidades de proc
                 def __updateSpecFromVoltage(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     try:
                         self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
                     except:
                         pass
                     self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
                     self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
                     self.dataOut.channelList = self.dataIn.channelList
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     self.dataOut.nBaud = self.dataIn.nBaud
                     self.dataOut.nCode = self.dataIn.nCode
                     self.dataOut.code = self.dataIn.code
                     self.dataOut.nProfiles = self.dataOut.nFFTPoints
                     self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
                     self.dataOut.utctime = self.firstdatatime
                     # asumo q la data esta decodificada
                     self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
                     # asumo q la data esta sin flip
                     self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.nIncohInt = 1
                     self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.realtime = self.dataIn.realtime
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.beam.codeList = self.dataIn.beam.codeList
                     self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
                     self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
                 def __getFft(self):
                     """
                     Convierte valores de Voltaje a Spectra
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.data_cspc
                         self.dataOut.data_dc
                         self.dataOut.heightList
                         self.profIndex
                         self.buffer
                         self.dataOut.flagNoData
                     """
                     fft_volt = numpy.fft.fft(
                         self.buffer, n=self.dataOut.nFFTPoints, axis=1)
                     fft_volt = fft_volt.astype(numpy.dtype('complex'))
                     dc = fft_volt[:, 0, :]
                     # calculo de self-spectra
                     fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
                     spc = fft_volt * numpy.conjugate(fft_volt)
                     spc = spc.real
                     blocksize = 0
                     blocksize += dc.size
                     blocksize += spc.size
                     cspc = None
                     pairIndex = 0
                     if self.dataOut.pairsList != None:
                         # calculo de cross-spectra
                         cspc = numpy.zeros(
                             (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
                         for pair in self.dataOut.pairsList:
                             if pair[0] not in self.dataOut.channelList:
                                 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
                                     str(pair), str(self.dataOut.channelList)))
                             if pair[1] not in self.dataOut.channelList:
                                 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
                                     str(pair), str(self.dataOut.channelList)))
                             cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
                                 numpy.conjugate(fft_volt[pair[1], :, :])
                             pairIndex += 1
                         blocksize += cspc.size
                     self.dataOut.data_spc = spc
                     self.dataOut.data_cspc = cspc
                     self.dataOut.data_dc = dc
                     self.dataOut.blockSize = blocksize
                     self.dataOut.flagShiftFFT = True
                 def run(self, nProfiles=None, nFFTPoints=None, pairsList=[], ippFactor=None, shift_fft=False):
                     if self.dataIn.type == "Spectra":
                         self.dataOut.copy(self.dataIn)
                         if shift_fft:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             shift = int(self.dataOut.nFFTPoints/2)
                             self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
                             if self.dataOut.data_cspc is not None:
                                 #desplaza a la derecha en el eje 2 determinadas posiciones
                                 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
                         return True
                     if self.dataIn.type == "Voltage":
                         self.dataOut.flagNoData = True
                         if nFFTPoints == None:
                             raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
                         if nProfiles == None:
                             nProfiles = nFFTPoints
                         if ippFactor == None:
                             ippFactor = 1
                         self.dataOut.ippFactor = ippFactor
                         self.dataOut.nFFTPoints = nFFTPoints
                         self.dataOut.pairsList = pairsList
                         if self.buffer is None:
                             self.buffer = numpy.zeros((self.dataIn.nChannels,
                                                        nProfiles,
                                                        self.dataIn.nHeights),
                                                       dtype='complex')
                         if self.dataIn.flagDataAsBlock:
                             nVoltProfiles = self.dataIn.data.shape[1]
                             if nVoltProfiles == nProfiles:
                                 self.buffer = self.dataIn.data.copy()
                                 self.profIndex = nVoltProfiles
                             elif nVoltProfiles < nProfiles:
                                 if self.profIndex == 0:
                                     self.id_min = 0
                                     self.id_max = nVoltProfiles
                                 self.buffer[:, self.id_min:self.id_max,
                                             :] = self.dataIn.data
                                 self.profIndex += nVoltProfiles
                                 self.id_min += nVoltProfiles
                                 self.id_max += nVoltProfiles
                             else:
                                 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
                                     self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
                                 self.dataOut.flagNoData = True
                                 return 0
                         else:
                             self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
                             self.profIndex += 1
                         if self.firstdatatime == None:
                             self.firstdatatime = self.dataIn.utctime
                         if self.profIndex == nProfiles:
                             self.__updateSpecFromVoltage()
                             self.__getFft()
                             self.dataOut.flagNoData = False
                             self.firstdatatime = None
                             self.profIndex = 0
                         return True
                     raise ValueError("The type of input object '%s' is not valid" % (
                         self.dataIn.type))
                 def __selectPairs(self, pairsList):
                     if not pairsList:
                         return
                     pairs = []
                     pairsIndex = []
                     for pair in pairsList:
                         if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
                             continue
                         pairs.append(pair)
                         pairsIndex.append(pairs.index(pair))
                     self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
                     self.dataOut.pairsList = pairs
                     return
                 def __selectPairsByChannel(self, channelList=None):
                     if channelList == None:
                         return
                     pairsIndexListSelected = []
                     for pairIndex in self.dataOut.pairsIndexList:
                         # First pair
                         if self.dataOut.pairsList[pairIndex][0] not in channelList:
                             continue
                         # Second pair
                         if self.dataOut.pairsList[pairIndex][1] not in channelList:
                             continue
                         pairsIndexListSelected.append(pairIndex)
                     if not pairsIndexListSelected:
                         self.dataOut.data_cspc = None
                         self.dataOut.pairsList = []
                         return
                     self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
                     self.dataOut.pairsList = [self.dataOut.pairsList[i]
                                               for i in pairsIndexListSelected]
                     return
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         if channel not in self.dataOut.channelList:
                             raise ValueError("Error selecting channels, Channel %d is not valid.\nAvailable channels = %s" % (
                                 channel, str(self.dataOut.channelList)))
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             raise ValueError("Error selecting channels: The value %d in channelIndexList is not valid.\nAvailable channel indexes = " % (
                                 channelIndex, self.dataOut.channelIndexList))
-            #         nChannels = len(channelIndexList)
                     data_spc = self.dataOut.data_spc[channelIndexList, :]
                     data_dc = self.dataOut.data_dc[channelIndexList, :]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_dc = data_dc
-                    self.dataOut.channelList = [
+                    # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
-                        self.dataOut.channelList[i] for i in channelIndexList]
+                    self.dataOut.channelList = range(len(channelIndexList))
-            #        self.dataOut.nChannels = nChannels
+                    self.__selectPairsByChannel(channelIndexList)
-                    self.__selectPairsByChannel(self.dataOut.channelList)
                     return 1
                 def selectFFTs(self, minFFT, maxFFT ):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
                     minFFT<= FFT <= maxFFT
                     """
                     if (minFFT > maxFFT):
                         raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
                     if (minFFT < self.dataOut.getFreqRange()[0]):
                         minFFT = self.dataOut.getFreqRange()[0]
                     if (maxFFT > self.dataOut.getFreqRange()[-1]):
                         maxFFT = self.dataOut.getFreqRange()[-1]
                     minIndex = 0
                     maxIndex = 0
                     FFTs = self.dataOut.getFreqRange()
                     inda = numpy.where(FFTs >= minFFT)
                     indb = numpy.where(FFTs <= maxFFT)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(FFTs)
                     self.selectFFTsByIndex(minIndex, maxIndex)
                     return 1
                 def setH0(self, h0, deltaHeight = None):
                     if not deltaHeight:
                         deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     nHeights = self.dataOut.nHeights
                     newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
                     self.dataOut.heightList = newHeiRange
                 def selectHeights(self, minHei, maxHei):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
                     minHei <= height <= maxHei
                     Input:
                         minHei    :    valor minimo de altura a considerar
                         maxHei    :    valor maximo de altura a considerar
                     Affected:
                         Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minHei > maxHei):
                         raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minHei, maxHei))
                     if (minHei < self.dataOut.heightList[0]):
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]):
                         maxHei = self.dataOut.heightList[-1]
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
                     newheis = numpy.where(
                         self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
                     if hei_ref != None:
                         newheis = numpy.where(self.dataOut.heightList > hei_ref)
                     minIndex = min(newheis[0])
                     maxIndex = max(newheis[0])
                     data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
                     heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
                     # determina indices
                     nheis = int(self.dataOut.radarControllerHeaderObj.txB /
                                 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
                     avg_dB = 10 * \
                         numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
                     beacon_dB = numpy.sort(avg_dB)[-nheis:]
                     beacon_heiIndexList = []
                     for val in avg_dB.tolist():
                         if val >= beacon_dB[0]:
                             beacon_heiIndexList.append(avg_dB.tolist().index(val))
                     #data_spc = data_spc[:,:,beacon_heiIndexList]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
                         #data_cspc = data_cspc[:,:,beacon_heiIndexList]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
                         #data_dc = data_dc[:,beacon_heiIndexList]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.heightList = heightList
                     self.dataOut.beacon_heiIndexList = beacon_heiIndexList
                     return 1
                 def selectFFTsByIndex(self, minIndex, maxIndex):
                     """
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nProfiles):
                         maxIndex = self.dataOut.nProfiles-1
                     #Spectra
                     data_spc = self.dataOut.data_spc[:,minIndex:maxIndex+1,:]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:,minIndex:maxIndex+1,:]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[minIndex:maxIndex+1,:]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
                     self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
                     self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
                     return 1
                 def selectHeightsByIndex(self, minIndex, maxIndex):
                     """
                     Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
                     minIndex <= index <= maxIndex
                     Input:
                         minIndex    :    valor de indice minimo de altura a considerar
                         maxIndex    :    valor de indice maximo de altura a considerar
                     Affected:
                         self.dataOut.data_spc
                         self.dataOut.data_cspc
                         self.dataOut.data_dc
                         self.dataOut.heightList
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
                             minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights - 1
                     # Spectra
                     data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
                     data_cspc = None
                     if self.dataOut.data_cspc is not None:
                         data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
                     data_dc = None
                     if self.dataOut.data_dc is not None:
                         data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
                     return 1
                 def removeDC(self, mode=2):
                     jspectra = self.dataOut.data_spc
                     jcspectra = self.dataOut.data_cspc
                     num_chan = jspectra.shape[0]
                     num_hei = jspectra.shape[2]
                     if jcspectra is not None:
                         jcspectraExist = True
                         num_pairs = jcspectra.shape[0]
                     else:
                         jcspectraExist = False
                     freq_dc = int(jspectra.shape[1] / 2)
                     ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
                     ind_vel = ind_vel.astype(int)
                     if ind_vel[0] < 0:
                         ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
                     if mode == 1:
                         jspectra[:, freq_dc, :] = (
                             jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2  # CORRECCION
                         if jcspectraExist:
                             jcspectra[:, freq_dc, :] = (
                                 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
                     if mode == 2:
                         vel = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for fil in range(4):
                             xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0, :]
                         for ich in range(num_chan):
                             yy = jspectra[ich, ind_vel, :]
                             jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
                             junkid = jspectra[ich, freq_dc, :] <= 0
                             cjunkid = sum(junkid)
                             if cjunkid.any():
                                 jspectra[ich, freq_dc, junkid.nonzero()] = (
                                     jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
                         if jcspectraExist:
                             for ip in range(num_pairs):
                                 yy = jcspectra[ip, ind_vel, :]
                                 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
                     self.dataOut.data_spc = jspectra
                     self.dataOut.data_cspc = jcspectra
                     return 1
                 def removeInterference2(self):
                     cspc = self.dataOut.data_cspc
                     spc = self.dataOut.data_spc
                     Heights = numpy.arange(cspc.shape[2])
                     realCspc = numpy.abs(cspc)
                     for i in range(cspc.shape[0]):
                         LinePower= numpy.sum(realCspc[i], axis=0)
                         Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
                         SelectedHeights = Heights[ numpy.where( LinePower < Threshold ) ]
                         InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
                         InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
                         InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
                         InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
                         #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
                         if len(InterferenceRange)<int(cspc.shape[1]*0.3):
                             cspc[i,InterferenceRange,:] = numpy.NaN
                     self.dataOut.data_cspc = cspc
                 def removeInterference(self,  interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None):
                     jspectra = self.dataOut.data_spc
                     jcspectra = self.dataOut.data_cspc
                     jnoise = self.dataOut.getNoise()
                     num_incoh = self.dataOut.nIncohInt
                     num_channel = jspectra.shape[0]
                     num_prof = jspectra.shape[1]
                     num_hei = jspectra.shape[2]
                     # hei_interf
                     if hei_interf is None:
                         count_hei = int(num_hei / 2)
                         hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
                         hei_interf = numpy.asarray(hei_interf)[0]
                     # nhei_interf
                     if (nhei_interf == None):
                         nhei_interf = 5
                     if (nhei_interf < 1):
                         nhei_interf = 1
                     if (nhei_interf > count_hei):
                         nhei_interf = count_hei
                     if (offhei_interf == None):
                         offhei_interf = 0
                     ind_hei = list(range(num_hei))
             #         mask_prof = numpy.asarray(range(num_prof - 2)) + 1
             #         mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
                     mask_prof = numpy.asarray(list(range(num_prof)))
                     num_mask_prof = mask_prof.size
                     comp_mask_prof = [0, num_prof / 2]
                     # noise_exist:    Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
                     if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
                         jnoise = numpy.nan
                     noise_exist = jnoise[0] < numpy.Inf
                     # Subrutina de Remocion de la Interferencia
                     for ich in range(num_channel):
                         # Se ordena los espectros segun su potencia (menor a mayor)
                         power = jspectra[ich, mask_prof, :]
                         power = power[:, hei_interf]
                         power = power.sum(axis=0)
                         psort = power.ravel().argsort()
                         # Se estima la interferencia promedio en los Espectros de Potencia empleando
                         junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
                             offhei_interf, nhei_interf + offhei_interf))]]]
                         if noise_exist:
                             #    tmp_noise = jnoise[ich] / num_prof
                             tmp_noise = jnoise[ich]
                         junkspc_interf = junkspc_interf - tmp_noise
                         #junkspc_interf[:,comp_mask_prof] = 0
                         jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
                         jspc_interf = jspc_interf.transpose()
                         # Calculando el espectro de interferencia promedio
                         noiseid = numpy.where(
                             jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
                         noiseid = noiseid[0]
                         cnoiseid = noiseid.size
                         interfid = numpy.where(
                             jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
                         interfid = interfid[0]
                         cinterfid = interfid.size
                         if (cnoiseid > 0):
                             jspc_interf[noiseid] = 0
                         # Expandiendo los perfiles a limpiar
                         if (cinterfid > 0):
                             new_interfid = (
                                 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
                             new_interfid = numpy.asarray(new_interfid)
                             new_interfid = {x for x in new_interfid}
                             new_interfid = numpy.array(list(new_interfid))
                             new_cinterfid = new_interfid.size
                         else:
                             new_cinterfid = 0
                         for ip in range(new_cinterfid):
                             ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
                             jspc_interf[new_interfid[ip]
                                         ] = junkspc_interf[ind[nhei_interf // 2], new_interfid[ip]]
                         jspectra[ich, :, ind_hei] = jspectra[ich, :,
                                                              ind_hei] - jspc_interf  # Corregir indices
                         # Removiendo la interferencia del punto de mayor interferencia
                         ListAux = jspc_interf[mask_prof].tolist()
                         maxid = ListAux.index(max(ListAux))
                         if cinterfid > 0:
                             for ip in range(cinterfid * (interf == 2) - 1):
                                 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
                                        (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
                                 cind = len(ind)
                                 if (cind > 0):
                                     jspectra[ich, interfid[ip], ind] = tmp_noise * \
                                         (1 + (numpy.random.uniform(cind) - 0.5) /
                                          numpy.sqrt(num_incoh))
                             ind = numpy.array([-2, -1, 1, 2])
                             xx = numpy.zeros([4, 4])
                             for id1 in range(4):
                                 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
                             xx_inv = numpy.linalg.inv(xx)
                             xx = xx_inv[:, 0]
                             ind = (ind + maxid + num_mask_prof) % num_mask_prof
                             yy = jspectra[ich, mask_prof[ind], :]
                             jspectra[ich, mask_prof[maxid], :] = numpy.dot(
                                 yy.transpose(), xx)
                         indAux = (jspectra[ich, :, :] < tmp_noise *
                                   (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
                         jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
                             (1 - 1 / numpy.sqrt(num_incoh))
                     # Remocion de Interferencia en el Cross Spectra
                     if jcspectra is None:
                         return jspectra, jcspectra
                     num_pairs = int(jcspectra.size / (num_prof * num_hei))
                     jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
                     for ip in range(num_pairs):
                         #-------------------------------------------
                         cspower = numpy.abs(jcspectra[ip, mask_prof, :])
                         cspower = cspower[:, hei_interf]
                         cspower = cspower.sum(axis=0)
                         cspsort = cspower.ravel().argsort()
                         junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
                             offhei_interf, nhei_interf + offhei_interf))]]]
                         junkcspc_interf = junkcspc_interf.transpose()
                         jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
                         ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
                         median_real = int(numpy.median(numpy.real(
                             junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
                         median_imag = int(numpy.median(numpy.imag(
                             junkcspc_interf[mask_prof[ind[list(range(3 * num_prof // 4))]], :])))
                         comp_mask_prof = [int(e) for e in comp_mask_prof]
                         junkcspc_interf[comp_mask_prof, :] = numpy.complex(
                             median_real, median_imag)
                         for iprof in range(num_prof):
                             ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
                             jcspc_interf[iprof] = junkcspc_interf[iprof, ind[nhei_interf // 2]]
                         # Removiendo la Interferencia
                         jcspectra[ip, :, ind_hei] = jcspectra[ip,
                                                               :, ind_hei] - jcspc_interf
                         ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
                         maxid = ListAux.index(max(ListAux))
                         ind = numpy.array([-2, -1, 1, 2])
                         xx = numpy.zeros([4, 4])
                         for id1 in range(4):
                             xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
                         xx_inv = numpy.linalg.inv(xx)
                         xx = xx_inv[:, 0]
                         ind = (ind + maxid + num_mask_prof) % num_mask_prof
                         yy = jcspectra[ip, mask_prof[ind], :]
                         jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
                     # Guardar Resultados
                     self.dataOut.data_spc = jspectra
                     self.dataOut.data_cspc = jcspectra
                     return 1
                 def setRadarFrequency(self, frequency=None):
                     if frequency != None:
                         self.dataOut.frequency = frequency
                     return 1
                 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
                     # validacion de rango
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
                         print('minHei: %.2f is out of the heights range' % (minHei))
                         print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
                         print('maxHei: %.2f is out of the heights range' % (maxHei))
                         print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
                         maxHei = self.dataOut.heightList[-1]
                     # validacion de velocidades
                     velrange = self.dataOut.getVelRange(1)
                     if minVel == None:
                         minVel = velrange[0]
                     if maxVel == None:
                         maxVel = velrange[-1]
                     if (minVel < velrange[0]) or (minVel > maxVel):
                         print('minVel: %.2f is out of the velocity range' % (minVel))
                         print('minVel is setting to %.2f' % (velrange[0]))
                         minVel = velrange[0]
                     if (maxVel > velrange[-1]) or (maxVel < minVel):
                         print('maxVel: %.2f is out of the velocity range' % (maxVel))
                         print('maxVel is setting to %.2f' % (velrange[-1]))
                         maxVel = velrange[-1]
                     # seleccion de indices para rango
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("some value in (%d,%d) is not valid" % (
                             minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights - 1
                     # seleccion de indices para velocidades
                     indminvel = numpy.where(velrange >= minVel)
                     indmaxvel = numpy.where(velrange <= maxVel)
                     try:
                         minIndexVel = indminvel[0][0]
                     except:
                         minIndexVel = 0
                     try:
                         maxIndexVel = indmaxvel[0][-1]
                     except:
                         maxIndexVel = len(velrange)
                     # seleccion del espectro
                     data_spc = self.dataOut.data_spc[:,
                                                      minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
                     # estimacion de ruido
                     noise = numpy.zeros(self.dataOut.nChannels)
                     for channel in range(self.dataOut.nChannels):
                         daux = data_spc[channel, :, :]
                         noise[channel] = hildebrand_sekhon(daux, self.dataOut.nIncohInt)
                     self.dataOut.noise_estimation = noise.copy()
                     return 1
             class IncohInt(Operation):
                 __profIndex = 0
                 __withOverapping = False
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer_spc = None
                 __buffer_cspc = None
                 __buffer_dc = None
                 __dataReady = False
                 __timeInterval = None
                 n = None
                 def __init__(self):
                     Operation.__init__(self)
                 def setup(self, n=None, timeInterval=None, overlapping=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n        :    Number of coherent integrations
                         timeInterval   :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping    :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer_spc = 0
                     self.__buffer_cspc = 0
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     self.__dataReady = False
                     self.__byTime = False
                     if n is None and timeInterval is None:
                         raise ValueError("n or timeInterval should be specified ...")
                     if n is not None:
                         self.n = int(n)
                     else:
                         self.__integrationtime = int(timeInterval)
                         self.n = None
                         self.__byTime = True
                 def putData(self, data_spc, data_cspc, data_dc):
                     """
                     Add a profile to the __buffer_spc and increase in one the __profileIndex
                     """
                     self.__buffer_spc += data_spc
                     if data_cspc is None:
                         self.__buffer_cspc = None
                     else:
                         self.__buffer_cspc += data_cspc
                     if data_dc is None:
                         self.__buffer_dc = None
                     else:
                         self.__buffer_dc += data_dc
                     self.__profIndex += 1
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     data_spc = self.__buffer_spc
                     data_cspc = self.__buffer_cspc
                     data_dc = self.__buffer_dc
                     n = self.__profIndex
                     self.__buffer_spc = 0
                     self.__buffer_cspc = 0
                     self.__buffer_dc = 0
                     self.__profIndex = 0
                     return data_spc, data_cspc, data_dc, n
                 def byProfiles(self, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if self.__profIndex == self.n:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def byTime(self, datatime, *args):
                     self.__dataReady = False
                     avgdata_spc = None
                     avgdata_cspc = None
                     avgdata_dc = None
                     self.putData(*args)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata_spc, avgdata_cspc, avgdata_dc
                 def integrate(self, datatime, *args):
                     if self.__profIndex == 0:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
                             datatime, *args)
                     else:
                         avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
                     if not self.__dataReady:
                         return None, None, None, None
                     return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
                 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
                     if n == 1:
                         return
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         self.setup(n, timeInterval, overlapping)
                         self.isConfig = True
                     avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
                                                                                         dataOut.data_spc,
                                                                                         dataOut.data_cspc,
                                                                                         dataOut.data_dc)
                     if self.__dataReady:
                         dataOut.data_spc = avgdata_spc
                         dataOut.data_cspc = avgdata_cspc
                         dataOut.data_dc = avgdata_dc
                         dataOut.nIncohInt *= self.n
                         dataOut.utctime = avgdatatime
                         dataOut.flagNoData = False
                     return dataOut

schainpy/model/proc/jroproc_voltage.py +3 -3

             import sys
             import numpy
             from scipy import interpolate
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Voltage
             from schainpy.utils import log
             from time import time
             @MPDecorator
             class VoltageProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Voltage()
                     self.flip = 1
                     self.setupReq = False
                 def run(self):
                     if self.dataIn.type == 'AMISR':
                         self.__updateObjFromAmisrInput()
                     if self.dataIn.type == 'Voltage':
                         self.dataOut.copy(self.dataIn)
                 #         self.dataOut.copy(self.dataIn)
                 def __updateObjFromAmisrInput(self):
                     self.dataOut.timeZone = self.dataIn.timeZone
                     self.dataOut.dstFlag = self.dataIn.dstFlag
                     self.dataOut.errorCount = self.dataIn.errorCount
                     self.dataOut.useLocalTime = self.dataIn.useLocalTime
                     self.dataOut.flagNoData = self.dataIn.flagNoData
                     self.dataOut.data = self.dataIn.data
                     self.dataOut.utctime = self.dataIn.utctime
                     self.dataOut.channelList = self.dataIn.channelList
                     #self.dataOut.timeInterval = self.dataIn.timeInterval
                     self.dataOut.heightList = self.dataIn.heightList
                     self.dataOut.nProfiles = self.dataIn.nProfiles
                     self.dataOut.nCohInt = self.dataIn.nCohInt
                     self.dataOut.ippSeconds = self.dataIn.ippSeconds
                     self.dataOut.frequency = self.dataIn.frequency
                     self.dataOut.azimuth = self.dataIn.azimuth
                     self.dataOut.zenith = self.dataIn.zenith
                     self.dataOut.beam.codeList = self.dataIn.beam.codeList
                     self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
                     self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
                 #
                 #        pass#
                 #
                 #    def init(self):
                 #
                 #
                 #        if self.dataIn.type == 'AMISR':
                 #            self.__updateObjFromAmisrInput()
                 #
                 #        if self.dataIn.type == 'Voltage':
                 #            self.dataOut.copy(self.dataIn)
                 #        # No necesita copiar en cada init() los atributos de dataIn
                 #        # la copia deberia hacerse por cada nuevo bloque de datos
                 def selectChannels(self, channelList):
                     channelIndexList = []
                     for channel in channelList:
                         if channel not in self.dataOut.channelList:
                             raise ValueError("Channel %d is not in %s" %(channel, str(self.dataOut.channelList)))
                         index = self.dataOut.channelList.index(channel)
                         channelIndexList.append(index)
                     self.selectChannelsByIndex(channelIndexList)
                 def selectChannelsByIndex(self, channelIndexList):
                     """
                     Selecciona un bloque de datos en base a canales segun el channelIndexList
                     Input:
                         channelIndexList    :    lista sencilla de canales a seleccionar por ej. [2,3,7]
                     Affected:
                         self.dataOut.data
                         self.dataOut.channelIndexList
                         self.dataOut.nChannels
                         self.dataOut.m_ProcessingHeader.totalSpectra
                         self.dataOut.systemHeaderObj.numChannels
                         self.dataOut.m_ProcessingHeader.blockSize
                     Return:
                         None
                     """
                     for channelIndex in channelIndexList:
                         if channelIndex not in self.dataOut.channelIndexList:
                             print(channelIndexList)
                             raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         data = self.dataOut.data[channelIndexList,:,:]
                     else:
                         data = self.dataOut.data[channelIndexList,:]
                     self.dataOut.data = data
-                    self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
+                    # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
-                    #        self.dataOut.nChannels = nChannels
+                    self.dataOut.channelList = range(len(channelIndexList))
                     return 1
                 def selectHeights(self, minHei=None, maxHei=None):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
                     minHei <= height <= maxHei
                     Input:
                         minHei    :    valor minimo de altura a considerar
                         maxHei    :    valor maximo de altura a considerar
                     Affected:
                         Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if minHei == None:
                         minHei = self.dataOut.heightList[0]
                     if maxHei == None:
                         maxHei = self.dataOut.heightList[-1]
                     if (minHei < self.dataOut.heightList[0]):
                         minHei = self.dataOut.heightList[0]
                     if (maxHei > self.dataOut.heightList[-1]):
                         maxHei = self.dataOut.heightList[-1]
                     minIndex = 0
                     maxIndex = 0
                     heights = self.dataOut.heightList
                     inda = numpy.where(heights >= minHei)
                     indb = numpy.where(heights <= maxHei)
                     try:
                         minIndex = inda[0][0]
                     except:
                         minIndex = 0
                     try:
                         maxIndex = indb[0][-1]
                     except:
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def selectHeightsByIndex(self, minIndex, maxIndex):
                     """
                     Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
                     minIndex <= index <= maxIndex
                     Input:
                         minIndex    :    valor de indice minimo de altura a considerar
                         maxIndex    :    valor de indice maximo de altura a considerar
                     Affected:
                         self.dataOut.data
                         self.dataOut.heightList
                     Return:
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                         raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
                     if (maxIndex >= self.dataOut.nHeights):
                         maxIndex = self.dataOut.nHeights
                     #voltage
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         data = self.dataOut.data[:,:, minIndex:maxIndex]
                     else:
                         data = self.dataOut.data[:, minIndex:maxIndex]
                     #         firstHeight = self.dataOut.heightList[minIndex]
                     self.dataOut.data = data
                     self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
                     if self.dataOut.nHeights <= 1:
                         raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
                     return 1
                 def filterByHeights(self, window):
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     if window == None:
                         window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
                     newdelta = deltaHeight * window
                     r = self.dataOut.nHeights % window
                     newheights = (self.dataOut.nHeights-r)/window
                     if newheights <= 1:
                         raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window))
                     if self.dataOut.flagDataAsBlock:
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         buffer = self.dataOut.data[:, :, 0:int(self.dataOut.nHeights-r)]
                         buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
                         buffer = numpy.sum(buffer,3)
                     else:
                         buffer = self.dataOut.data[:,0:int(self.dataOut.nHeights-r)]
                         buffer = buffer.reshape(self.dataOut.nChannels,int(self.dataOut.nHeights/window),int(window))
                         buffer = numpy.sum(buffer,2)
                     self.dataOut.data = buffer
                     self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
                     self.dataOut.windowOfFilter = window
                 def setH0(self, h0, deltaHeight = None):
                     if not deltaHeight:
                         deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     nHeights = self.dataOut.nHeights
                     newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
                     self.dataOut.heightList = newHeiRange
                 def deFlip(self, channelList = []):
                     data = self.dataOut.data.copy()
                     if self.dataOut.flagDataAsBlock:
                         flip = self.flip
                         profileList = list(range(self.dataOut.nProfiles))
                         if not channelList:
                             for thisProfile in profileList:
                                 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
                                 flip *= -1.0
                         else:
                             for thisChannel in channelList:
                                 if thisChannel not in self.dataOut.channelList:
                                     continue
                                 for thisProfile in profileList:
                                     data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
                                     flip *= -1.0
                         self.flip = flip
                     else:
                         if not channelList:
                             data[:,:] = data[:,:]*self.flip
                         else:
                             for thisChannel in channelList:
                                 if thisChannel not in self.dataOut.channelList:
                                     continue
                                 data[thisChannel,:] = data[thisChannel,:]*self.flip
                         self.flip *= -1.
                     self.dataOut.data = data
                 def setRadarFrequency(self, frequency=None):
                     if frequency != None:
                         self.dataOut.frequency = frequency
                     return 1
                 def interpolateHeights(self, topLim, botLim):
                     #69 al 72 para julia
                     #82-84 para meteoros
                     if len(numpy.shape(self.dataOut.data))==2:
                         sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
                         sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
                         #self.dataOut.data[:,botLim:limSup+1] = sampInterp
                         self.dataOut.data[:,botLim:topLim+1] = sampInterp
                     else:
                         nHeights = self.dataOut.data.shape[2]
                         x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
                         y = self.dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
                         f = interpolate.interp1d(x, y, axis = 2)
                         xnew = numpy.arange(botLim,topLim+1)
                         ynew = f(xnew)
                         self.dataOut.data[:,:,botLim:topLim+1]  = ynew
                 # import collections
             class CohInt(Operation):
                 isConfig = False
                 __profIndex = 0
                 __byTime = False
                 __initime = None
                 __lastdatatime = None
                 __integrationtime = None
                 __buffer = None
                 __bufferStride = []
                 __dataReady = False
                 __profIndexStride = 0
                 __dataToPutStride = False
                 n = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     #   self.isConfig = False
                 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
                     """
                     Set the parameters of the integration class.
                     Inputs:
                         n               :    Number of coherent integrations
                         timeInterval    :    Time of integration. If the parameter "n" is selected this one does not work
                         overlapping     :
                     """
                     self.__initime = None
                     self.__lastdatatime = 0
                     self.__buffer = None
                     self.__dataReady = False
                     self.byblock = byblock
                     self.stride = stride
                     if n == None and timeInterval == None:
                         raise ValueError("n or timeInterval should be specified ...")
                     if n != None:
                         self.n = n
                         self.__byTime = False
                     else:
                         self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
                         self.n = 9999
                         self.__byTime = True
                     if overlapping:
                         self.__withOverlapping = True
                         self.__buffer = None
                     else:
                         self.__withOverlapping = False
                         self.__buffer = 0
                     self.__profIndex = 0
                 def putData(self, data):
                     """
                     Add a profile to the __buffer and increase in one the __profileIndex
                     """
                     if not self.__withOverlapping:
                         self.__buffer += data.copy()
                         self.__profIndex += 1
                         return
                     #Overlapping data
                     nChannels, nHeis = data.shape
                     data = numpy.reshape(data, (1, nChannels, nHeis))
                     #If the buffer is empty then it takes the data value
                     if self.__buffer is None:
                         self.__buffer = data
                         self.__profIndex += 1
                         return
                     #If the buffer length is lower than n then stakcing the data value
                     if self.__profIndex < self.n:
                         self.__buffer = numpy.vstack((self.__buffer, data))
                         self.__profIndex += 1
                         return
                     #If the buffer length is equal to n then replacing the last buffer value with the data value
                     self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
                     self.__buffer[self.n-1] = data
                     self.__profIndex = self.n
                     return
                 def pushData(self):
                     """
                     Return the sum of the last profiles and the profiles used in the sum.
                     Affected:
                     self.__profileIndex
                     """
                     if not self.__withOverlapping:
                         data = self.__buffer
                         n = self.__profIndex
                         self.__buffer = 0
                         self.__profIndex = 0
                         return data, n
                     #Integration with Overlapping
                     data = numpy.sum(self.__buffer, axis=0)
                     # print data
                     # raise
                     n = self.__profIndex
                     return data, n
                 def byProfiles(self, data):
                     self.__dataReady = False
                     avgdata = None
                     #         n = None
                     # print data
                     # raise
                     self.putData(data)
                     if self.__profIndex == self.n:
                         avgdata, n = self.pushData()
                         self.__dataReady = True
                     return avgdata
                 def byTime(self, data, datatime):
                     self.__dataReady = False
                     avgdata = None
                     n = None
                     self.putData(data)
                     if (datatime - self.__initime) >= self.__integrationtime:
                         avgdata, n = self.pushData()
                         self.n = n
                         self.__dataReady = True
                     return avgdata
                 def integrateByStride(self, data, datatime):
                     # print data
                     if self.__profIndex == 0:
                         self.__buffer = [[data.copy(), datatime]]
                     else:
                         self.__buffer.append([data.copy(),datatime])
                     self.__profIndex += 1
                     self.__dataReady = False
                     if self.__profIndex == self.n * self.stride :
                         self.__dataToPutStride = True
                         self.__profIndexStride = 0
                         self.__profIndex = 0
                         self.__bufferStride = []
                         for i in range(self.stride):
                             current = self.__buffer[i::self.stride]
                             data = numpy.sum([t[0] for t in current], axis=0)
                             avgdatatime = numpy.average([t[1] for t in current])
                             # print data
                             self.__bufferStride.append((data, avgdatatime))
                     if self.__dataToPutStride:
                         self.__dataReady = True
                         self.__profIndexStride += 1
                         if self.__profIndexStride == self.stride:
                             self.__dataToPutStride = False
                         # print self.__bufferStride[self.__profIndexStride - 1]
                         # raise
                         return self.__bufferStride[self.__profIndexStride - 1]
                     return None, None
                 def integrate(self, data, datatime=None):
                     if self.__initime == None:
                         self.__initime = datatime
                     if self.__byTime:
                         avgdata = self.byTime(data, datatime)
                     else:
                         avgdata = self.byProfiles(data)
                     self.__lastdatatime = datatime
                     if avgdata is None:
                         return None, None
                     avgdatatime = self.__initime
                     deltatime = datatime - self.__lastdatatime
                     if not self.__withOverlapping:
                         self.__initime = datatime
                     else:
                         self.__initime += deltatime
                     return avgdata, avgdatatime
                 def integrateByBlock(self, dataOut):
                     times = int(dataOut.data.shape[1]/self.n)
                     avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
                     id_min = 0
                     id_max = self.n
                     for i in range(times):
                         junk = dataOut.data[:,id_min:id_max,:]
                         avgdata[:,i,:] = junk.sum(axis=1)
                         id_min += self.n
                         id_max += self.n
                     timeInterval = dataOut.ippSeconds*self.n
                     avgdatatime = (times - 1) * timeInterval + dataOut.utctime
                     self.__dataReady = True
                     return avgdata, avgdatatime
                 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
                     if not self.isConfig:
                         self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
                         self.isConfig = True
                     if dataOut.flagDataAsBlock:
                         """
                         Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         avgdata, avgdatatime = self.integrateByBlock(dataOut)
                         dataOut.nProfiles /= self.n
                     else:
                         if stride is None:
                             avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
                         else:
                             avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
                     #   dataOut.timeInterval *= n
                     dataOut.flagNoData = True
                     if self.__dataReady:
                         dataOut.data = avgdata
                         dataOut.nCohInt *= self.n
                         dataOut.utctime = avgdatatime
                         # print avgdata, avgdatatime
                         # raise
                         #   dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
                         dataOut.flagNoData = False
                     return dataOut
             class Decoder(Operation):
                 isConfig = False
                 __profIndex = 0
                 code = None
                 nCode = None
                 nBaud = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.times = None
                     self.osamp = None
                 #         self.__setValues = False
                     self.isConfig = False
                     self.setupReq = False
                 def setup(self, code, osamp, dataOut):
                     self.__profIndex = 0
                     self.code = code
                     self.nCode = len(code)
                     self.nBaud = len(code[0])
                     if (osamp != None) and (osamp >1):
                         self.osamp = osamp
                         self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
                         self.nBaud = self.nBaud*self.osamp
                     self.__nChannels = dataOut.nChannels
                     self.__nProfiles = dataOut.nProfiles
                     self.__nHeis = dataOut.nHeights
                     if self.__nHeis < self.nBaud:
                         raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
                     #Frequency
                     __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
                     __codeBuffer[:,0:self.nBaud] = self.code
                     self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
                     if dataOut.flagDataAsBlock:
                         self.ndatadec = self.__nHeis #- self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
                     else:
                         #Time
                         self.ndatadec = self.__nHeis #- self.nBaud + 1
                         self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
                 def __convolutionInFreq(self, data):
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=1)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=1)
                     return data
                 def __convolutionInFreqOpt(self, data):
                     raise NotImplementedError
                 def __convolutionInTime(self, data):
                     code = self.code[self.__profIndex]
                     for i in range(self.__nChannels):
                         self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInTime(self, data):
                     repetitions = self.__nProfiles / self.nCode
                     junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
                     junk = junk.flatten()
                     code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
                     profilesList = range(self.__nProfiles)
                     for i in range(self.__nChannels):
                         for j in profilesList:
                             self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInFreq(self, data):
                     raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
                     fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
                     fft_data = numpy.fft.fft(data, axis=2)
                     conv = fft_data*fft_code
                     data = numpy.fft.ifft(conv,axis=2)
                     return data
                 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
                     if dataOut.flagDecodeData:
                         print("This data is already decoded, recoding again ...")
                     if not self.isConfig:
                         if code is None:
                             if dataOut.code is None:
                                 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
                             code = dataOut.code
                         else:
                             code = numpy.array(code).reshape(nCode,nBaud)
                         self.setup(code, osamp, dataOut)
                         self.isConfig = True
                         if mode == 3:
                             sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
                         if times != None:
                             sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
                     if self.code is None:
                         print("Fail decoding: Code is not defined.")
                         return
                     self.__nProfiles = dataOut.nProfiles
                     datadec = None
                     if mode == 3:
                         mode = 0
                     if dataOut.flagDataAsBlock:
                         """
                         Decoding when data have been read as block,
                         """
                         if mode == 0:
                             datadec = self.__convolutionByBlockInTime(dataOut.data)
                         if mode == 1:
                             datadec = self.__convolutionByBlockInFreq(dataOut.data)
                     else:
                         """
                         Decoding when data have been read profile by profile
                         """
                         if mode == 0:
                             datadec = self.__convolutionInTime(dataOut.data)
                         if mode == 1:
                             datadec = self.__convolutionInFreq(dataOut.data)
                         if mode == 2:
                             datadec = self.__convolutionInFreqOpt(dataOut.data)
                     if datadec is None:
                         raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
                     dataOut.code = self.code
                     dataOut.nCode = self.nCode
                     dataOut.nBaud = self.nBaud
                     dataOut.data = datadec
                     dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
                     dataOut.flagDecodeData = True #asumo q la data esta decodificada
                     if self.__profIndex == self.nCode-1:
                         self.__profIndex = 0
                         return dataOut
                     self.__profIndex += 1
                     return dataOut
                 #        dataOut.flagDeflipData = True #asumo q la data no esta sin flip
             class ProfileConcat(Operation):
                 isConfig = False
                 buffer = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                 def reset(self):
                     self.buffer = numpy.zeros_like(self.buffer)
                     self.start_index = 0
                     self.times = 1
                 def setup(self, data, m, n=1):
                     self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
                     self.nHeights = data.shape[1]#.nHeights
                     self.start_index = 0
                     self.times = 1
                 def concat(self, data):
                     self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
                     self.start_index = self.start_index + self.nHeights
                 def run(self, dataOut, m):
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         self.setup(dataOut.data, m, 1)
                         self.isConfig = True
                     if dataOut.flagDataAsBlock:
                         raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
                     else:
                         self.concat(dataOut.data)
                         self.times += 1
                         if self.times > m:
                             dataOut.data = self.buffer
                             self.reset()
                             dataOut.flagNoData = False
                             # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
                             deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                             xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
                             dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
                             dataOut.ippSeconds *= m
                     return dataOut
             class ProfileSelector(Operation):
                 profileIndex = None
                 # Tamanho total de los perfiles
                 nProfiles = None
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                 def incProfileIndex(self):
                     self.profileIndex += 1
                     if self.profileIndex >= self.nProfiles:
                         self.profileIndex = 0
                 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
                     if profileIndex < minIndex:
                         return False
                     if profileIndex > maxIndex:
                         return False
                     return True
                 def isThisProfileInList(self, profileIndex, profileList):
                     if profileIndex not in profileList:
                         return False
                     return True
                 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
                     """
                     ProfileSelector:
                     Inputs:
                         profileList        :    Index of profiles selected. Example: profileList = (0,1,2,7,8)
                         profileRangeList    :    Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
                         rangeList            :    List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
                     """
                     if rangeList is not None:
                         if type(rangeList[0]) not in (tuple, list):
                             rangeList = [rangeList]
                     dataOut.flagNoData = True
                     if dataOut.flagDataAsBlock:
                         """
                         data dimension  = [nChannels, nProfiles, nHeis]
                         """
                         if profileList != None:
                             dataOut.data = dataOut.data[:,profileList,:]
                         if profileRangeList != None:
                             minIndex = profileRangeList[0]
                             maxIndex = profileRangeList[1]
                             profileList = list(range(minIndex, maxIndex+1))
                             dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
                         if rangeList != None:
                             profileList = []
                             for thisRange in rangeList:
                                 minIndex = thisRange[0]
                                 maxIndex = thisRange[1]
                                 profileList.extend(list(range(minIndex, maxIndex+1)))
                             dataOut.data = dataOut.data[:,profileList,:]
                         dataOut.nProfiles = len(profileList)
                         dataOut.profileIndex = dataOut.nProfiles - 1
                         dataOut.flagNoData = False
                         return dataOut
                     """
                     data dimension  = [nChannels, nHeis]
                     """
                     if profileList != None:
                         if self.isThisProfileInList(dataOut.profileIndex, profileList):
                             self.nProfiles = len(profileList)
                             dataOut.nProfiles = self.nProfiles
                             dataOut.profileIndex = self.profileIndex
                             dataOut.flagNoData = False
                             self.incProfileIndex()
                         return dataOut
                     if profileRangeList != None:
                         minIndex = profileRangeList[0]
                         maxIndex = profileRangeList[1]
                         if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
                             self.nProfiles = maxIndex - minIndex + 1
                             dataOut.nProfiles = self.nProfiles
                             dataOut.profileIndex = self.profileIndex
                             dataOut.flagNoData = False
                             self.incProfileIndex()
                         return dataOut
                     if rangeList != None:
                         nProfiles = 0
                         for thisRange in rangeList:
                             minIndex = thisRange[0]
                             maxIndex = thisRange[1]
                             nProfiles += maxIndex - minIndex + 1
                         for thisRange in rangeList:
                             minIndex = thisRange[0]
                             maxIndex = thisRange[1]
                             if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
                                 self.nProfiles = nProfiles
                                 dataOut.nProfiles = self.nProfiles
                                 dataOut.profileIndex = self.profileIndex
                                 dataOut.flagNoData = False
                                 self.incProfileIndex()
                                 break
                         return dataOut
                     if beam != None: #beam is only for AMISR data
                         if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
                             dataOut.flagNoData = False
                             dataOut.profileIndex = self.profileIndex
                             self.incProfileIndex()
                         return dataOut
                     raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
                     #return False
                     return dataOut
             class Reshaper(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.__buffer = None
                     self.__nitems = 0
                 def __appendProfile(self, dataOut, nTxs):
                     if self.__buffer is None:
                         shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
                         self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
                     ini = dataOut.nHeights * self.__nitems
                     end = ini + dataOut.nHeights
                     self.__buffer[:, ini:end] = dataOut.data
                     self.__nitems += 1
                     return int(self.__nitems*nTxs)
                 def __getBuffer(self):
                     if self.__nitems == int(1./self.__nTxs):
                         self.__nitems = 0
                         return self.__buffer.copy()
                     return None
                 def __checkInputs(self, dataOut, shape, nTxs):
                     if shape is None and nTxs is None:
                         raise ValueError("Reshaper: shape of factor should be defined")
                     if nTxs:
                         if nTxs < 0:
                             raise ValueError("nTxs should be greater than 0")
                         if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
                             raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
                         shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
                         return shape, nTxs
                     if len(shape) != 2 and len(shape) !=  3:
                         raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
                     if len(shape) == 2:
                         shape_tuple = [dataOut.nChannels]
                         shape_tuple.extend(shape)
                     else:
                         shape_tuple = list(shape)
                     nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
                     return shape_tuple, nTxs
                 def run(self, dataOut, shape=None, nTxs=None):
                     shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
                     else:
                         if self.__nTxs < 1:
                             self.__appendProfile(dataOut, self.__nTxs)
                             new_data = self.__getBuffer()
                             if new_data is not None:
                                 dataOut.data = new_data
                                 dataOut.flagNoData = False
                                 profileIndex = dataOut.profileIndex*nTxs
                         else:
                             raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds /= self.__nTxs
                     return dataOut
             class SplitProfiles(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                 def run(self, dataOut, n):
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         #nchannels, nprofiles, nsamples
                         shape = dataOut.data.shape
                         if shape[2] % n != 0:
                             raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
                         new_shape = shape[0], shape[1]*n, int(shape[2]/n)
                         dataOut.data = numpy.reshape(dataOut.data, new_shape)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles/n) - 1
                     else:
                         raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles*n)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds /= n
                     return dataOut
             class CombineProfiles(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.__remData = None
                     self.__profileIndex = 0
                 def run(self, dataOut, n):
                     dataOut.flagNoData = True
                     profileIndex = None
                     if dataOut.flagDataAsBlock:
                         #nchannels, nprofiles, nsamples
                         shape = dataOut.data.shape
                         new_shape = shape[0], shape[1]/n, shape[2]*n
                         if shape[1] % n != 0:
                             raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
                         dataOut.data = numpy.reshape(dataOut.data, new_shape)
                         dataOut.flagNoData = False
                         profileIndex = int(dataOut.nProfiles*n) - 1
                     else:
                         #nchannels, nsamples
                         if self.__remData is None:
                             newData = dataOut.data
                         else:
                             newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
                         self.__profileIndex += 1
                         if self.__profileIndex < n:
                             self.__remData = newData
                             #continue
                             return
                         self.__profileIndex = 0
                         self.__remData = None
                         dataOut.data = newData
                         dataOut.flagNoData = False
                         profileIndex = dataOut.profileIndex/n
                     deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
                     dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
                     dataOut.nProfiles = int(dataOut.nProfiles/n)
                     dataOut.profileIndex = profileIndex
                     dataOut.ippSeconds *= n
                     return dataOut
             # import collections
             # from scipy.stats import mode
             #
             # class Synchronize(Operation):
             #
             #     isConfig = False
             #     __profIndex = 0
             #
             #     def __init__(self, **kwargs):
             #
             #         Operation.__init__(self, **kwargs)
             # #         self.isConfig = False
             #         self.__powBuffer = None
             #         self.__startIndex = 0
             #         self.__pulseFound = False
             #
             #     def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
             #
             #         #Read data
             #
             #         powerdB = dataOut.getPower(channel = channel)
             #         noisedB = dataOut.getNoise(channel = channel)[0]
             #
             #         self.__powBuffer.extend(powerdB.flatten())
             #
             #         dataArray = numpy.array(self.__powBuffer)
             #
             #         filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
             #
             #         maxValue = numpy.nanmax(filteredPower)
             #
             #         if maxValue < noisedB + 10:
             #             #No se encuentra ningun pulso de transmision
             #             return None
             #
             #         maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
             #
             #         if len(maxValuesIndex) < 2:
             #             #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
             #             return None
             #
             #         phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
             #
             #         #Seleccionar solo valores con un espaciamiento de nSamples
             #         pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
             #
             #         if len(pulseIndex) < 2:
             #             #Solo se encontro un pulso de transmision con ancho mayor a 1
             #             return None
             #
             #         spacing = pulseIndex[1:] - pulseIndex[:-1]
             #
             #         #remover senales que se distancien menos de 10 unidades o muestras
             #         #(No deberian existir IPP menor a 10 unidades)
             #
             #         realIndex = numpy.where(spacing > 10 )[0]
             #
             #         if len(realIndex) < 2:
             #             #Solo se encontro un pulso de transmision con ancho mayor a 1
             #             return None
             #
             #         #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
             #         realPulseIndex = pulseIndex[realIndex]
             #
             #         period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
             #
             #         print "IPP = %d samples" %period
             #
             #         self.__newNSamples = dataOut.nHeights #int(period)
             #         self.__startIndex = int(realPulseIndex[0])
             #
             #         return 1
             #
             #
             #     def setup(self, nSamples, nChannels, buffer_size = 4):
             #
             #         self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
             #                                           maxlen = buffer_size*nSamples)
             #
             #         bufferList = []
             #
             #         for i in range(nChannels):
             #             bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) +  numpy.NAN,
             #                                           maxlen = buffer_size*nSamples)
             #
             #             bufferList.append(bufferByChannel)
             #
             #         self.__nSamples = nSamples
             #         self.__nChannels = nChannels
             #         self.__bufferList = bufferList
             #
             #     def run(self, dataOut, channel = 0):
             #
             #         if not self.isConfig:
             #             nSamples = dataOut.nHeights
             #             nChannels = dataOut.nChannels
             #             self.setup(nSamples, nChannels)
             #             self.isConfig = True
             #
             #         #Append new data to internal buffer
             #         for thisChannel in range(self.__nChannels):
             #             bufferByChannel = self.__bufferList[thisChannel]
             #             bufferByChannel.extend(dataOut.data[thisChannel])
             #
             #         if self.__pulseFound:
             #             self.__startIndex -= self.__nSamples
             #
             #         #Finding Tx Pulse
             #         if not self.__pulseFound:
             #             indexFound = self.__findTxPulse(dataOut, channel)
             #
             #             if indexFound == None:
             #                 dataOut.flagNoData = True
             #                 return
             #
             #             self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
             #             self.__pulseFound = True
             #             self.__startIndex = indexFound
             #
             #         #If pulse was found ...
             #         for thisChannel in range(self.__nChannels):
             #             bufferByChannel = self.__bufferList[thisChannel]
             #             #print self.__startIndex
             #             x = numpy.array(bufferByChannel)
             #             self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
             #
             #         deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
             #         dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
             # #             dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
             #
             #         dataOut.data = self.__arrayBuffer
             #
             #         self.__startIndex += self.__newNSamples
             #
             #         return

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages