schain Commit - r1594:16c6f41ffb3e · Jicamarca Repository

Remove numpy.float, numpy.int, numpy.complex

Juan C. Espinoza -

r1594:16c6f41ffb3e

parent child

Context file:

r1594:16c6f41ffb3e

Collapse all files

schainpy/model/io/jroIO_digitalRF.py +3 -3

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             # SUBCHANNELS EN VEZ DE CHANNELS
             # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
             # ACTUALIZACION DE VERSION
             # HEADERS
             # MODULO DE ESCRITURA
             # METADATA
             import os
             import time
             import datetime
             import numpy
             import timeit
             from fractions import Fraction
             from time import time
             from time import sleep
             import schainpy.admin
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             import pickle
             try:
                 import digital_rf
             except:
                 pass
             class DigitalRFReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.dataOut = Voltage()
                     self.__printInfo = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__codeType = 0
                     self.__ippKm = None
                     self.__nCode = None
                     self.__nBaud = None
                     self.__code = None
                     self.dtype = None
                     self.oldAverage = None
                     self.path = None
                 def close(self):
                     print('Average of writing to digital rf format is ', self.oldAverage * 1000)
                     return
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample / self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
                     nProfiles = 1.0 / ippSeconds  # Number of profiles in one second
                     try:
                         self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
                             self.__radarControllerHeader)
                     except:
                         self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
                             txA=0,
                             txB=0,
                             nWindows=1,
                             nHeights=self.__nSamples,
                             firstHeight=self.__firstHeigth,
                             deltaHeight=self.__deltaHeigth,
                             codeType=self.__codeType,
                             nCode=self.__nCode, nBaud=self.__nBaud,
                             code=self.__code)
                     try:
                         self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
                     except:
                         self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                     nProfiles=nProfiles,
                                                                     nChannels=len(
                                                                         self.__channelList),
                                                                     adcResolution=14)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = self.dtype
                     # self.dataOut.nChannels = 0
                     # self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = int(nProfiles)
                     self.dataOut.heightList = self.__firstHeigth + \
-                        numpy.arange(self.__nSamples, dtype=numpy.float) * \
+                        numpy.arange(self.__nSamples, dtype=numpy.float32) * \
                         self.__deltaHeigth
                     self.dataOut.channelList = list(range(self.__num_subchannels))
                     self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
                     # self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     self.dataOut.flagDataAsBlock = False
                     # Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     # timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.timeZone = self.__timezone / 60
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     try:
                         self.dataOut.nCohInt = self.fixed_metadata_dict.get(
                             'nCohInt', self.nCohInt)
                         # asumo que la data esta decodificada
                         self.dataOut.flagDecodeData = self.fixed_metadata_dict.get(
                             'flagDecodeData', self.flagDecodeData)
                         # asumo que la data esta sin flip
                         self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData']
                         self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT']
                         self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime']
                     except:
                         pass
                     self.dataOut.ippSeconds = ippSeconds
                     # Time interval between profiles
                     # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
                 def findDatafiles(self, path, startDate=None, endDate=None):
                     if not os.path.isdir(path):
                         return []
                     try:
                         digitalReadObj = digital_rf.DigitalRFReader(
                             path, load_all_metadata=True)
                     except:
                         digitalReadObj = digital_rf.DigitalRFReader(path)
                     channelNameList = digitalReadObj.get_channels()
                     if not channelNameList:
                         return []
                     metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
                     sample_rate = metadata_dict['sample_rate'][0]
                     this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
                     try:
                         timezone = this_metadata_file['timezone'].value
                     except:
                         timezone = 0
                     startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(
                         channelNameList[0]) / sample_rate - timezone
                     startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
                     endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
                     if not startDate:
                         startDate = startDatetime.date()
                     if not endDate:
                         endDate = endDatatime.date()
                     dateList = []
                     thisDatetime = startDatetime
                     while(thisDatetime <= endDatatime):
                         thisDate = thisDatetime.date()
                         if thisDate < startDate:
                             continue
                         if thisDate > endDate:
                             break
                         dateList.append(thisDate)
                         thisDatetime += datetime.timedelta(1)
                     return dateList
                 def setup(self, path=None,
                           startDate=None,
                           endDate=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           channelList=None,
                           nSamples=None,
                           online=False,
                           delay=60,
                           buffer_size=1024,
                           ippKm=None,
                           nCohInt=1,
                           nCode=1,
                           nBaud=1,
                           flagDecodeData=False,
-                          code=numpy.ones((1, 1), dtype=numpy.int),
+                          code=numpy.ones((1, 1), dtype=numpy.int32),
                           **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
                         delay
                     '''
                     self.path = path
                     self.nCohInt = nCohInt
                     self.flagDecodeData = flagDecodeData
                     self.i = 0
                     if not os.path.isdir(path):
                         raise ValueError("[Reading] Directory %s does not exist" % path)
                     try:
                         self.digitalReadObj = digital_rf.DigitalRFReader(
                             path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf.DigitalRFReader(path)
                     channelNameList = self.digitalReadObj.get_channels()
                     if not channelNameList:
                         raise ValueError("[Reading] Directory %s does not have any files" % path)
                     if not channelList:
                         channelList = list(range(len(channelNameList)))
                     ##########  Reading metadata ######################
                     top_properties = self.digitalReadObj.get_properties(
                         channelNameList[channelList[0]])
                     self.__num_subchannels = top_properties['num_subchannels']
                     self.__sample_rate = 1.0 * \
                         top_properties['sample_rate_numerator'] / \
                         top_properties['sample_rate_denominator']
                     # self.__samples_per_file = top_properties['samples_per_file'][0]
                     self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate  # why 0.15?
                     this_metadata_file = self.digitalReadObj.get_digital_metadata(
                         channelNameList[channelList[0]])
                     metadata_bounds = this_metadata_file.get_bounds()
                     self.fixed_metadata_dict = this_metadata_file.read(
                         metadata_bounds[0])[metadata_bounds[0]]  # GET FIRST HEADER
                     try:
                         self.__processingHeader = self.fixed_metadata_dict['processingHeader']
                         self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
                         self.__systemHeader = self.fixed_metadata_dict['systemHeader']
                         self.dtype = pickle.loads(self.fixed_metadata_dict['dtype'])
                     except:
                         pass
                     self.__frequency = None
                     self.__frequency = self.fixed_metadata_dict.get('frequency', 1)
                     self.__timezone = self.fixed_metadata_dict.get('timezone', 18000)
                     try:
                         nSamples = self.fixed_metadata_dict['nSamples']
                     except:
                         nSamples = None
                     self.__firstHeigth = 0
                     try:
                         codeType = self.__radarControllerHeader['codeType']
                     except:
                         codeType = 0
                     try:
                         if codeType:
                             nCode = self.__radarControllerHeader['nCode']
                             nBaud = self.__radarControllerHeader['nBaud']
                             code = self.__radarControllerHeader['code']
                     except:
                         pass
                     if not ippKm:
                         try:
                             # seconds to km
                             ippKm = self.__radarControllerHeader['ipp']
                         except:
                             ippKm = None
                     ####################################################
                     self.__ippKm = ippKm
                     startUTCSecond = None
                     endUTCSecond = None
                     if startDate:
                         startDatetime = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (
                             startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond = (endDatetime - datetime.datetime(1970,
 , 1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(
                         channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index / self.__sample_rate
                     if start_index > startUTCSecond * self.__sample_rate:
                         startUTCSecond = start_index / self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond = end_index / self.__sample_rate
                     if end_index < endUTCSecond * self.__sample_rate:
                         endUTCSecond = end_index / self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError("[Reading] nSamples or ippKm should be defined")
                         nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
                     channelBoundList = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName = channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(
                             thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
                     self.i = 0
                     self.__delay = delay
                     self.__codeType = codeType
                     self.__nCode = nCode
                     self.__nBaud = nBaud
                     self.__code = code
                     self.__datapath = path
                     self.__online = online
                     self.__channelList = channelList
                     self.__channelNameList = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples = nSamples
                     self.__samples_to_read = int(nSamples)  # FIJO: AHORA 40
                     self.__nChannels = len(self.__channelList)
                     self.__startUTCSecond = startUTCSecond
                     self.__endUTCSecond = endUTCSecond
                     self.__timeInterval = 1.0 * self.__samples_to_read / \
                         self.__sample_rate  # Time interval
                     if online:
                         # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     # por que en el otro metodo lo primero q se hace es sumar samplestoread
                     self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read
                     self.__data_buffer = numpy.zeros(
-                        (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex)
+                        (self.__num_subchannels, self.__samples_to_read), dtype=complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print("[Reading] Digital RF Data was found from %s to %s " % (
                         datetime.datetime.utcfromtimestamp(
                             self.__startUTCSecond - self.__timezone),
                         datetime.datetime.utcfromtimestamp(
                             self.__endUTCSecond - self.__timezone)
                     ))
                     print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
                                                                         datetime.datetime.utcfromtimestamp(
                         endUTCSecond - self.__timezone)
                     ))
                     self.oldAverage = None
                     self.count = 0
                     self.executionTime = 0
                 def __reload(self):
                     #         print
                     #         print "%s not in range [%s, %s]" %(
                     #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                     #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                     #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                     #                                           )
                     print("[Reading] reloading metadata ...")
                     try:
                         self.digitalReadObj.reload(complete_update=True)
                     except:
                         self.digitalReadObj = digital_rf.DigitalRFReader(self.path)
                     start_index, end_index = self.digitalReadObj.get_bounds(
                         self.__channelNameList[self.__channelList[0]])
                     if start_index > self.__startUTCSecond * self.__sample_rate:
                         self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
                     if end_index > self.__endUTCSecond * self.__sample_rate:
                         self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
                         print()
                         print("[Reading] New timerange found [%s, %s] " % (
                             datetime.datetime.utcfromtimestamp(
                                 self.__startUTCSecond - self.__timezone),
                             datetime.datetime.utcfromtimestamp(
                                 self.__endUTCSecond - self.__timezone)
                         ))
                         return True
                     return False
                 def timeit(self, toExecute):
                     t0 = time.time()
                     toExecute()
                     self.executionTime = time.time() - t0
                     if self.oldAverage is None:
                         self.oldAverage = self.executionTime
                     self.oldAverage = (self.executionTime + self.count *
                                        self.oldAverage) / (self.count + 1.0)
                     self.count = self.count + 1.0
                     return
                 def __readNextBlock(self, seconds=30, volt_scale=1):
                     '''
                     '''
                     # Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample += self.__samples_to_read
                     if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                         print ("[Reading] There are no more data into selected time-range")
                         if self.__online:
                             sleep(3)
                             self.__reload()
                         else:
                             return False
                         if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                             return False
                             self.__thisUnixSample -= self.__samples_to_read
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:  # TODO VARIOS CHANNELS?
                         for indexSubchannel in range(self.__num_subchannels):
                             try:
                                 t0 = time()
                                 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                               self.__samples_to_read,
                                                                               thisChannelName, sub_channel=indexSubchannel)
                                 self.executionTime = time() - t0
                                 if self.oldAverage is None:
                                     self.oldAverage = self.executionTime
                                 self.oldAverage = (
                                     self.executionTime + self.count * self.oldAverage) / (self.count + 1.0)
                                 self.count = self.count + 1.0
                             except IOError as e:
                                 # read next profile
                                 self.__flagDiscontinuousBlock = True
                                 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
                                 break
                             if result.shape[0] != self.__samples_to_read:
                                 self.__flagDiscontinuousBlock = True
                                 print("[Reading] %s: Too few samples were found, just %d/%d  samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                                          result.shape[0],
                                                                                                          self.__samples_to_read))
                                 break
                             self.__data_buffer[indexSubchannel, :] = result * volt_scale
                             indexChannel += 1
                             dataOk = True
                     self.__utctime = self.__thisUnixSample / self.__sample_rate
                     if not dataOk:
                         return False
                     print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                   self.__samples_to_read,
                                                                   self.__timeInterval))
                     self.__bufferIndex = 0
                     return True
                 def __isBufferEmpty(self):
                     return self.__bufferIndex > self.__samples_to_read - self.__nSamples  # 40960 - 40
                 def getData(self, seconds=30, nTries=5):
                     '''
                         This method gets the data from files and put the data into the dataOut object
                         In addition, increase el the buffer counter in one.
                         Return:
                             data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                         buffer. Si no hay mas archivos a leer retorna None.
                         Affected:
                             self.dataOut
                             self.profileIndex
                             self.flagDiscontinuousBlock
                             self.flagIsNewBlock
                     '''
                     # print("getdata")
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         # print("hi")
                         self.__flagDiscontinuousBlock = False
                         while True:
                             # print ("q ha pasado")
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
                                 raise schainpy.admin.SchainError('Error')
                                 return
                             if self.__flagDiscontinuousBlock:
                                 raise schainpy.admin.SchainError('discontinuous block found')
                                 return
                             if not self.__online:
                                 raise schainpy.admin.SchainError('Online?')
                                 return
                             err_counter += 1
                             if err_counter > nTries:
                                 raise schainpy.admin.SchainError('Max retrys reach')
                                 return
                             print('[Reading] waiting %d seconds to read a new block' % seconds)
                             time.sleep(seconds)
                     self.dataOut.data = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
                     self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.dataOut.profileIndex = self.profileIndex
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex += 1
                     if self.profileIndex == self.dataOut.nProfiles:
                         self.profileIndex = 0
                     return True
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
                     # self.systemHeaderObj.printInfo()
                     # self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     return
                     # print self.profileIndex
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                     # self.i = self.i+1
                     self.getData(seconds=self.__delay)
                     return
             @MPDecorator
             class DigitalRFWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     Operation.__init__(self, **kwargs)
                     self.metadata_dict = {}
                     self.dataOut = None
                     self.dtype = None
                     self.oldAverage = 0
                 def setHeader(self):
                     self.metadata_dict['frequency'] = self.dataOut.frequency
                     self.metadata_dict['timezone'] = self.dataOut.timeZone
                     self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype)
                     self.metadata_dict['nProfiles'] = self.dataOut.nProfiles
                     self.metadata_dict['heightList'] = self.dataOut.heightList
                     self.metadata_dict['channelList'] = self.dataOut.channelList
                     self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
                     self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
                     self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT
                     self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime
                     self.metadata_dict['nCohInt'] = self.dataOut.nCohInt
                     self.metadata_dict['type'] = self.dataOut.type
                     self.metadata_dict['flagDataAsBlock'] = getattr(
                         self.dataOut, 'flagDataAsBlock', None)  # chequear
                 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataOut: Input data will also be outputa data
                     '''
                     self.setHeader()
                     self.__ippSeconds = dataOut.ippSeconds
                     self.__deltaH = dataOut.getDeltaH()
                     self.__sample_rate = 1e6 * 0.15 / self.__deltaH
                     self.__dtype = dataOut.dtype
                     if len(dataOut.dtype) == 2:
                         self.__dtype = dataOut.dtype[0]
                     self.__nSamples = dataOut.systemHeaderObj.nSamples
                     self.__nProfiles = dataOut.nProfiles
                     if self.dataOut.type != 'Voltage':
                         raise 'Digital RF cannot be used with this data type'
                         self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len(
                             self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
                     else:
                         self.arr_data = numpy.ones((self.__nSamples, len(
                             self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
                     file_cadence_millisecs = 1000
                     sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator()
                     sample_rate_numerator = int(sample_rate_fraction.numerator)
                     sample_rate_denominator = int(sample_rate_fraction.denominator)
                     start_global_index = dataOut.utctime * self.__sample_rate
                     uuid = 'prueba'
                     compression_level = 0
                     checksum = False
                     is_complex = True
                     num_subchannels = len(dataOut.channelList)
                     is_continuous = True
                     marching_periods = False
                     self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
                                                                       fileCadence, start_global_index,
                                                                       sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
                                                                       is_complex, num_subchannels, is_continuous, marching_periods)
                     metadata_dir = os.path.join(path, 'metadata')
                     os.system('mkdir %s' % (metadata_dir))
                     self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1,  # 236, file_cadence_millisecs / 1000
                                                                                     sample_rate_numerator, sample_rate_denominator,
                                                                                     metadataFile)
                     self.isConfig = True
                     self.currentSample = 0
                     self.oldAverage = 0
                     self.count = 0
                     return
                 def writeMetadata(self):
                     start_idx = self.__sample_rate * self.dataOut.utctime
                     self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict(
                     )
                     self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict(
                     )
                     self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict(
                     )
                     self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
                     return
                 def timeit(self, toExecute):
                     t0 = time()
                     toExecute()
                     self.executionTime = time() - t0
                     if self.oldAverage is None:
                         self.oldAverage = self.executionTime
                     self.oldAverage = (self.executionTime + self.count *
                                        self.oldAverage) / (self.count + 1.0)
                     self.count = self.count + 1.0
                     return
                 def writeData(self):
                     if self.dataOut.type != 'Voltage':
                         raise 'Digital RF cannot be used with this data type'
                         for channel in self.dataOut.channelList:
                             for i in range(self.dataOut.nFFTPoints):
                                 self.arr_data[1][channel * self.dataOut.nFFTPoints +
                                                  i]['r'] = self.dataOut.data[channel][i].real
                                 self.arr_data[1][channel * self.dataOut.nFFTPoints +
                                                  i]['i'] = self.dataOut.data[channel][i].imag
                     else:
                         for i in range(self.dataOut.systemHeaderObj.nSamples):
                             for channel in self.dataOut.channelList:
                                 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
                                 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
                     def f(): return self.digitalWriteObj.rf_write(self.arr_data)
                     self.timeit(f)
                     return
                 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataOut: object with the data
                     '''
                     # print dataOut.__dict__
                     self.dataOut = dataOut
                     if not self.isConfig:
                         self.setup(dataOut, path, frequency, fileCadence,
                                    dirCadence, metadataCadence, **kwargs)
                         self.writeMetadata()
                     self.writeData()
                     # # self.currentSample += 1
                     # if self.dataOut.flagDataAsBlock or self.currentSample == 1:
                     # self.writeMetadata()
                     # # if self.currentSample == self.__nProfiles: self.currentSample = 0
                     return dataOut  # en la version 2.7 no aparece este return
                 def close(self):
                     print('[Writing] - Closing files ')
                     print('Average of writing to digital rf format is ', self.oldAverage * 1000)
                     try:
                         self.digitalWriteObj.close()
                     except:
                         pass

schainpy/model/io/jroIO_hf.py +2 -2

             '''
             Created on Jul 3, 2014
             @author: roj-com0419
             '''
             import os, sys
             import time, datetime
             import h5py
             import numpy
             import fnmatch
             import re
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             def isNumber(str):
                 """
                 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                 Excepciones:
                     Si un determinado string no puede ser convertido a numero
                 Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                 Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                 """
                 try:
                     float(str)
                     return True
                 except:
                     return False
             def getFileFromSet(path, ext, set=None):
                 validFilelist = []
                 fileList = os.listdir(path)
                 if len(fileList) < 1:
                     return None
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         number = int(thisFile[6:16])
                  #       year = int(thisFile[1:5])
                  #       doy  = int(thisFile[5:8])
                     except:
                         continue
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if len(validFilelist) < 1:
                     return None
                 validFilelist = sorted(validFilelist, key=str.lower)
                 if set == None:
                     return validFilelist[-1]
                 print("set =" , set)
                 for thisFile in validFilelist:
                     if set <= int(thisFile[6:16]):
                         print(thisFile, int(thisFile[6:16]))
                         return thisFile
                 return validFilelist[-1]
                 myfile = fnmatch.filter(validFilelist, '*%10d*' % (set))
                 # myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
                 if len(myfile) != 0:
                     return myfile[0]
                 else:
                     filename = '*%10.10d%s' % (set, ext.lower())
                     print('the filename %s does not exist' % filename)
                     print('...going to the last file: ')
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             def getlastFileFromPath(path, ext):
                 """
             Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
                 al final de la depuracion devuelve el ultimo file de la lista que quedo.
                 Input:
                     fileList    :    lista conteniendo todos los files (sin path) que componen una determinada carpeta
                     ext         :    extension de los files contenidos en una carpeta
                 Return:
                     El ultimo file de una determinada carpeta, no se considera el path.
                 """
                 validFilelist = []
                 fileList = os.listdir(path)
                 # 0 1234 567 89A BCDE
                 # H YYYY DDD SSS .ext
                 for thisFile in fileList:
                     try:
                         number = int(thisFile[6:16])
                     except:
                         print("There is a file or folder with different format")
                     if not isNumber(number):
                         continue
             #        year = thisFile[1:5]
             #        if not isNumber(year):
             #            continue
             #       doy = thisFile[5:8]
             #        if not isNumber(doy):
             #            continue
                     number = int(number)
             #        year = int(year)
             #        doy = int(doy)
                     if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
                         continue
                     validFilelist.append(thisFile)
                 if validFilelist:
                     validFilelist = sorted(validFilelist, key=str.lower)
                     return validFilelist[-1]
                 return None
             class HFReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 path = None
                 startDate = None
                 endDate = None
                 startTime = None
                 endTime = None
                 walk = None
                 isConfig = False
                 dataOut = None
                 nTries = 3
                 ext = ".hdf5"
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self, **kwargs)
                     self.isConfig = False
                     self.datablock = None
                     self.filename_current = None
                     self.utc = 0
                     self.ext = '.hdf5'
                     self.flagIsNewFile = 1
                     #-------------------------------------------------
                     self.fileIndex = None
                     self.profileIndex_offset = None
                     self.filenameList = []
                     self.hfFilePointer = None
                     self.filename_online = None
                     self.status = True
                     self.flagNoMoreFiles = False
                     self.__waitForNewFile = 20
                     #--------------------------------------------------
                     self.dataOut = self.createObjByDefault()
                 def createObjByDefault(self):
                     dataObj = Voltage()
                     return dataObj
                 def setObjProperties(self):
                     pass
                 def getBlockDimension(self):
                     """
                     Obtiene la cantidad de puntos a leer por cada bloque de datos
                     Affected:
                         self.blocksize
                     Return:
                         None
                     """
                     pts2read = self.nChannels * self.nHeights * self.nProfiles
                     self.blocksize = pts2read
                 def __readHeader(self):
                     self.nProfiles = 100
                     self.nHeights = 1000
                     self.nChannels = 2
                     self.__firstHeigth = 0
                     self.__nSamples = 1000
                     self.__deltaHeigth = 1.5
                     self.__sample_rate = 1e5
                     # self.__frequency=2.72e6
                     # self.__frequency=3.64e6
                     self.__frequency = None
                     self.__online = False
                     self.filename_next_set = None
                     # print "Frequency of Operation:", self.__frequency
                 def __setParameters(self, path='', startDate='', endDate='', startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print('Path %s does not exits' % self.path)
                         return
                     return
                 def __selDates(self, hf_dirname_format):
                     try:
                         dir_hf_filename = self.path + "/" + hf_dirname_format
                         fp = h5py.File(dir_hf_filename, 'r')
                         hipoc = fp['t'].value
                         fp.close()
                         date_time = datetime.datetime.utcfromtimestamp(hipoc)
                         year = int(date_time[0:4])
                         month = int(date_time[5:7])
                         dom = int(date_time[8:10])
                         thisDate = datetime.date(year, month, dom)
                         if (thisDate >= self.startDate and thisDate <= self.endDate):
                             return hf_dirname_format
                     except:
                         return None
                 def __findDataForDates(self, online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat, x) for x in os.listdir(self.path)]
                     dirnameList = [x for x in dirnameList if x != None]
                     dirnameList = [x.string for x in dirnameList]
                     if not(online):
                         dirnameList = [self.__selDates(x) for x in dirnameList]
                         dirnameList = [x for x in dirnameList if x != None]
                     if len(dirnameList) > 0:
                             self.status = 1
                             self.dirnameList = dirnameList
                             self.dirnameList.sort()
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate, self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate, self.endTime)
                     print('Filtering Files from %s to %s' % (startDateTime_Reader, endDateTime_Reader))
                     print('........................................')
                     filter_filenameList = []
                     self.filenameList.sort()
                     for i in range(len(self.filenameList) - 1):
                         filename = self.filenameList[i]
                         dir_hf_filename = filename
                         fp = h5py.File(dir_hf_filename, 'r')
                         hipoc = fp['t'].value
                         hipoc = hipoc + self.timezone
                         date_time = datetime.datetime.utcfromtimestamp(hipoc)
                         fp.close()
                         year = int(date_time[0:4])
                         month = int(date_time[5:7])
                         dom = int(date_time[8:10])
                         hour = int(date_time[11:13])
                         min = int(date_time[14:16])
                         sec = int(date_time[17:19])
                         this_time = datetime.datetime(year, month, dom, hour, min, sec)
                         if (this_time >= startDateTime_Reader and this_time <= endDateTime_Reader):
                             filter_filenameList.append(filename)
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __getFilenameList(self):
                     # print "hola"
                     # print self.dirnameList
                     dirList = [os.path.join(self.path, x) for x in self.dirnameList]
                     self.filenameList = dirList
                     # print self.filenameList
                     # print "pase",len(self.filenameList)
                 def __selectDataForTimes(self, online=False):
                     if not(self.status):
                         return None
                     #----------------
                     self.__getFilenameList()
                     #----------------
                     if not(online):
                         if not(self.all):
                             self.__getTimeFromData()
                         if len(self.filenameList) > 0:
                             self.status = 1
                             self.filenameList.sort()
                         else:
                             self.status = 0
                             return None
                     else:
                         if self.set != None:
                             filename = getFileFromSet(self.path, self.ext, self.set)
                             if self.flag_nextfile == True:
                                 self.dirnameList = [filename]
                                 fullfilename = self.path + "/" + filename
                                 self.filenameList = [fullfilename]
                                 self.filename_next_set = int(filename[6:16]) + 10
                                 self.flag_nextfile = False
                             else:
                                 print(filename)
                                 print("PRIMERA CONDICION")
                                 # if self.filename_next_set== int(filename[6:16]):
                                 print("TODO BIEN")
                                 if filename == None:
                                     raise ValueError("corregir")
                                 self.dirnameList = [filename]
                                 fullfilename = self.path + "/" + filename
                                 self.filenameList = [fullfilename]
                                 self.filename_next_set = int(filename[6:16]) + 10
                                 print("Setting next file", self.filename_next_set)
                                 self.set = int(filename[6:16])
                                 if True:
                                     pass
                                 else:
                                     print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
                         else:
                             filename = getlastFileFromPath(self.path, self.ext)
                             if self.flag_nextfile == True:
                                 self.dirnameList = [filename]
                                 fullfilename = self.path + "/" + filename
                                 self.filenameList = [self.filenameList[-1]]
                                 self.filename_next_set = int(filename[6:16]) + 10
                                 self.flag_nextfile = False
                             else:
                                filename = getFileFromSet(self.path, self.ext, self.set)
                                print(filename)
                                print("PRIMERA CONDICION")
                                 # if self.filename_next_set== int(filename[6:16]):
                                print("TODO BIEN")
                                if filename == None:
                                    raise ValueError("corregir")
                                self.dirnameList = [filename]
                                fullfilename = self.path + "/" + filename
                                self.filenameList = [fullfilename]
                                self.filename_next_set = int(filename[6:16]) + 10
                                print("Setting next file", self.filename_next_set)
                                self.set = int(filename[6:16])
                                if True:
                                    pass
                                else:
                                    print("ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO")
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         ext,
                                         startTime=datetime.time(0, 0, 0),
                                         endTime=datetime.time(23, 59, 59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     # print self.dirnameList
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print("%s" % (self.filenameList[i]))
                     return
                 def searchFilesOnLine(self,
                                         path,
                                         expLabel="",
                                         ext=None,
                                         startDate=None,
                                         endDate=None,
                                         walk=True,
                                         set=None):
                     startDate = datetime.datetime.utcnow().date()
                     endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
                     self.__checkPath()
                     fullpath = path
                     print("%s folder was found: " % (fullpath))
                     if set == None:
                         self.set = None
                         filename = getlastFileFromPath(fullpath, ext)
                         startDate = datetime.datetime.utcnow().date
                         endDate = datetime.datetime.utcnow().date()
             #
                     else:
                         filename = getFileFromSet(fullpath, ext, set)
                         startDate = None
                         endDate = None
             #
                     if not (filename):
                         return None, None, None, None, None
                     # print "%s file was found" %(filename)
             #
             #         dir_hf_filename= self.path+"/"+filename
             #         fp= h5py.File(dir_hf_filename,'r')
             #         hipoc=fp['t'].value
             #         fp.close()
             #         date_time=datetime.datetime.utcfromtimestamp(hipoc)
             #
             #         year =int(date_time[0:4])
             #         month=int(date_time[5:7])
             #         dom  =int(date_time[8:10])
             #         set= int(filename[4:10])
             #         self.set=set-1
                     # self.dirnameList=[filename]
                     filenameList = fullpath + "/" + filename
                     self.dirnameList = [filename]
                     self.filenameList = [filenameList]
                     self.flag_nextfile = True
                     # self.__findDataForDates(online=True)
                     # self.dirnameList=[self.dirnameList[-1]]
                     # print self.dirnameList
                     # self.__selectDataForTimes(online=True)
                     # return fullpath,filename,year,month,dom,set
                     return
                 def __setNextFile(self, online=False):
                     """
                     """
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         return 0
                     return 1
                 def __setNextFileOffline(self):
                     """
                     """
                     idFile = self.fileIndex
                     while(True):
                         idFile += 1
                         if not (idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
                             return 0
                         filename = self.filenameList[idFile]
                         hfFilePointer = h5py.File(filename, 'r')
                         epoc = hfFilePointer['t'].value
                         # this_time=datetime.datetime(year,month,dom,hour,min,sec)
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.hfFilePointer = hfFilePointer
                     hfFilePointer.close()
                     self.__t0 = epoc
                     print("Setting the file: %s" % self.filename)
                     return 1
                 def __setNextFileOnline(self):
                     """
                     """
                     print("SOY NONE", self.set)
                     if self.set == None:
                         pass
                     else:
                         self.set += 10
                     filename = self.filenameList[0]  # fullfilename
                     if self.filename_online != None:
                         self.__selectDataForTimes(online=True)
                         filename = self.filenameList[0]
                         while self.filename_online == filename:
                             print('waiting %d seconds to get a new file...' % (self.__waitForNewFile))
                             time.sleep(self.__waitForNewFile)
                             # self.__findDataForDates(online=True)
                             self.set = self.filename_next_set
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                             sizeoffile = os.path.getsize(filename)
                     # print filename
                     sizeoffile = os.path.getsize(filename)
                     if sizeoffile < 1670240:
                         print("%s is not the rigth  size" % filename)
                         delay = 50
                         print('waiting %d seconds for delay...' % (delay))
                         time.sleep(delay)
                     sizeoffile = os.path.getsize(filename)
                     if sizeoffile < 1670240:
                         delay = 50
                         print('waiting %d  more seconds for delay...' % (delay))
                         time.sleep(delay)
                     sizeoffile = os.path.getsize(filename)
                     if sizeoffile < 1670240:
                         delay = 50
                         print('waiting %d  more seconds for delay...' % (delay))
                         time.sleep(delay)
                     try:
                         hfFilePointer = h5py.File(filename, 'r')
                     except:
                         print("Error reading file %s" % filename)
                     self.filename_online = filename
                     epoc = hfFilePointer['t'].value
                     self.hfFilePointer = hfFilePointer
                     hfFilePointer.close()
                     self.__t0 = epoc
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print("Setting the file: %s" % self.filename)
                     return 1
                 def __getExpParameters(self):
                     if not(self.status):
                         return None
                 def setup(self,
                            path=None,
                            startDate=None,
                            endDate=None,
                            startTime=datetime.time(0, 0, 0),
                            endTime=datetime.time(23, 59, 59),
                            set=None,
                            expLabel="",
                            ext=None,
                            all=0,
                            timezone=0,
                            online=False,
                            delay=60,
                            walk=True):
                     '''
                     In this method we should set all initial parameters.
                     '''
                     if path == None:
                         raise ValueError("The path is not valid")
                     if ext == None:
                         ext = self.ext
                     self.timezone = timezone
                     self.online = online
                     self.all = all
                     # if set==None:
                     # print set
                     if not(online):
                         print("Searching files in offline mode...")
                         self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
                     else:
                         print("Searching files in online mode...")
                         self.searchFilesOnLine(path, walk, ext, set=set)
                         if set == None:
                             pass
                         else:
                             self.set = set - 10
             #             for nTries in range(self.nTries):
             #
             #                 fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
             #
             #                 if fullpath:
             #                     break
             #                 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
             #                 time.sleep(self.delay)
             #             if not(fullpath):
             #                 print "There ins't valid files in %s" % path
             #                 return None
                     if not(self.filenameList):
                         print("There  is no files into the folder: %s" % (path))
                         sys.exit(-1)
                     self.__getExpParameters()
                     self.fileIndex = -1
                     self.__setNextFile(online)
                     self.__readMetadata()
                     self.__setLocalVariables()
                     self.__setHeaderDO()
                     # self.profileIndex_offset= 0
                     # self.profileIndex = self.profileIndex_offset
                     self.isConfig = True
                 def __readMetadata(self):
                     self.__readHeader()
                 def __setLocalVariables(self):
-                    self.datablock = numpy.zeros((self.nChannels, self.nHeights, self.nProfiles), dtype=numpy.complex)
+                    self.datablock = numpy.zeros((self.nChannels, self.nHeights, self.nProfiles), dtype=complex)
                     #
                     self.profileIndex = 9999
                 def __setHeaderDO(self):
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
                     self.dataOut.systemHeaderObj = SystemHeader()
                     #---------------------------------------------------------
                     self.dataOut.systemHeaderObj.nProfiles = 100
                     self.dataOut.systemHeaderObj.nSamples = 1000
                     SAMPLING_STRUCTURE = [('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
                     self.dataOut.radarControllerHeaderObj.samplingWindow = numpy.zeros((1,), SAMPLING_STRUCTURE)
                     self.dataOut.radarControllerHeaderObj.samplingWindow['h0'] = 0
                     self.dataOut.radarControllerHeaderObj.samplingWindow['dh'] = 1.5
                     self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'] = 1000
                     self.dataOut.radarControllerHeaderObj.nHeights = int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
                     self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
                     self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
                     self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
                     self.dataOut.radarControllerHeaderObj.nWindows = 1
                     self.dataOut.radarControllerHeaderObj.codetype = 0
                     self.dataOut.radarControllerHeaderObj.numTaus = 0
                     # self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
                     # self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
                     # self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
                     # self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
                     self.dataOut.radarControllerHeaderObj.code_size = 0
                     self.dataOut.nBaud = 0
                     self.dataOut.nCode = 0
                     self.dataOut.nPairs = 0
                     #---------------------------------------------------------
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     self.dataOut.nProfiles = 1
-                    self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float) * self.__deltaHeigth
+                    self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float32) * self.__deltaHeigth
                     self.dataOut.channelList = list(range(self.nChannels))
                     # self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     # Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     self.dataOut.timeZone = self.timezone
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
                     self.dataOut.flagDecodeData = False  # asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False  # asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
                     # Time interval between profiles
                     # self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.nProfiles:
                         return 1
                     return 0
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def __setNewBlock(self):
                     if self.hfFilePointer == None:
                         return 0
                     if self.flagIsNewFile:
                         return 1
                     if self.profileIndex < self.nProfiles:
                         return 1
                     self.__setNextFile(self.online)
                     return 1
                 def readBlock(self):
                     fp = h5py.File(self.filename, 'r')
                                   # Puntero que apunta al archivo hdf5
                     ch0 = (fp['ch0']).value  # Primer canal (100,1000)--(perfiles,alturas)
                     ch1 = (fp['ch1']).value  # Segundo canal (100,1000)--(perfiles,alturas)
                     fp.close()
                     ch0 = ch0.swapaxes(0, 1)  # Primer canal (100,1000)--(alturas,perfiles)
                     ch1 = ch1.swapaxes(0, 1)  # Segundo canal (100,1000)--(alturas,perfiles)
                     self.datablock = numpy.array([ch0, ch1])
                     self.flagIsNewFile = 0
                     self.profileIndex = 0
                     return 1
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     if self.__hasNotDataInBuffer():
                         if not(self.readNextBlock()):
                             self.dataOut.flagNodata = True
                             return 0
                     ##############################
                     ##############################
                     self.dataOut.data = self.datablock[:, :, self.profileIndex]
                     self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds * self.profileIndex
                     self.dataOut.profileIndex = self.profileIndex
                     self.dataOut.flagNoData = False
                     self.profileIndex += 1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

schainpy/model/io/jroIO_param.py +28 -14

             import os
             import time
             import datetime
             import numpy
             import h5py
             import schainpy.admin
             from schainpy.model.data.jrodata import *
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.io.jroIO_base import *
             from schainpy.utils import log
             class HDFReader(Reader, ProcessingUnit):
                 """Processing unit to read HDF5 format files
                 This unit reads HDF5 files created with `HDFWriter` operation contains
                 by default two groups Data and Metadata all variables would be saved as `dataOut`
                 attributes.
                 It is possible to read any HDF5 file by given the structure in the `description`
                 parameter, also you can add extra values to metadata with the parameter `extras`.
                 Parameters:
                 -----------
                 path : str
                     Path where files are located.
                 startDate : date
                     Start date of the files
                 endDate : list
                     End date of the files
                 startTime : time
                     Start time of the files
                 endTime : time
                     End time of the files
                 description : dict, optional
                     Dictionary with the description of the HDF5 file
                 extras : dict, optional
                     Dictionary with extra metadata to be be added to `dataOut`
                 Examples
                 --------
                 desc = {
                     'Data': {
                         'data_output': ['u', 'v', 'w'],
                         'utctime': 'timestamps',
                     }  ,
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 extras = {
                     'timeZone': 300
                 }
                 reader = project.addReadUnit(
                     name='HDFReader',
                     path='/path/to/files',
                     startDate='2019/01/01',
                     endDate='2019/01/31',
                     startTime='00:00:00',
                     endTime='23:59:59',
                     # description=json.dumps(desc),
                     # extras=json.dumps(extras),
                     )
                 """
                 __attrs__ = ['path', 'startDate', 'endDate', 'startTime', 'endTime', 'description', 'extras']
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.ext = ".hdf5"
                     self.optchar = "D"
                     self.meta = {}
                     self.data = {}
                     self.open_file = h5py.File
                     self.open_mode = 'r'
                     self.description = {}
                     self.extras = {}
                     self.filefmt = "*%Y%j***"
                     self.folderfmt = "*%Y%j"
                     self.utcoffset = 0
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                     return
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata()
                     self.__readData()
                     self.__setBlockList()
                     for attr in self.meta:
                         setattr(self.dataOut, attr, self.meta[attr])
                     self.blockIndex = 0
                     return
                 def __setBlockList(self):
                     '''
                     Selects the data within the times defined
                     self.fp
                     self.startTime
                     self.endTime
                     self.blockList
                     self.blocksPerFile
                     '''
                     startTime = self.startTime
                     endTime = self.endTime
                     thisUtcTime = self.data['utctime'] + self.utcoffset
                     self.interval = numpy.min(thisUtcTime[1:] - thisUtcTime[:-1])
                     thisDatetime = datetime.datetime.utcfromtimestamp(thisUtcTime[0])
                     thisDate = thisDatetime.date()
                     thisTime = thisDatetime.time()
                     startUtcTime = (datetime.datetime.combine(thisDate, startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     endUtcTime = (datetime.datetime.combine(thisDate, endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
                     ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
                     self.blockList = ind
                     self.blocksPerFile = len(ind)
                     return
                 def __readMetadata(self):
                     '''
                     Reads Metadata
                     '''
                     meta = {}
                     if self.description:
                         for key, value in self.description['Metadata'].items():
                             meta[key] = self.fp[value][()]
                     else:
                         grp = self.fp['Metadata']
                         for name in grp:
                             meta[name] = grp[name][()]
                     if self.extras:
                         for key, value in self.extras.items():
                             meta[key] = value
                     self.meta = meta
                     return
                 def __readData(self):
                     data = {}
                     if self.description:
                         for key, value in self.description['Data'].items():
                             if isinstance(value, str):
                                 if isinstance(self.fp[value], h5py.Dataset):
                                     data[key] = self.fp[value][()]
                                 elif isinstance(self.fp[value], h5py.Group):
                                     array = []
                                     for ch in self.fp[value]:
                                         array.append(self.fp[value][ch][()])
                                     data[key] = numpy.array(array)
                             elif isinstance(value, list):
                                 array = []
                                 for ch in value:
                                     array.append(self.fp[ch][()])
                                 data[key] = numpy.array(array)
                     else:
                         grp = self.fp['Data']
                         for name in grp:
                             if isinstance(grp[name], h5py.Dataset):
                                 array = grp[name][()]
                             elif isinstance(grp[name], h5py.Group):
                                 array = []
                                 for ch in grp[name]:
                                     array.append(grp[name][ch][()])
                                 array = numpy.array(array)
                             else:
                                 log.warning('Unknown type: {}'.format(name))
                             if name in self.description:
                                 key = self.description[name]
                             else:
                                 key = name
                             data[key] = array
                     self.data = data
                     return
                 def getData(self):
                     for attr in self.data:
                         if self.data[attr].ndim == 1:
                             setattr(self.dataOut, attr, self.data[attr][self.blockIndex])
                         else:
                             setattr(self.dataOut, attr, self.data[attr][:, self.blockIndex])
                     self.dataOut.flagNoData = False
                     self.blockIndex += 1
                     log.log("Block No. {}/{} -> {}".format(
                         self.blockIndex,
                         self.blocksPerFile,
                         self.dataOut.datatime.ctime()), self.name)
                     return
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     if self.blockIndex == self.blocksPerFile:
                         self.setNextFile()
                     self.getData()
                     if 'type' in self.meta:
                         self.dataOut.type = self.meta['type'].decode('utf-8')
                     return
             @MPDecorator
             class HDFWriter(Operation):
                 """Operation to write HDF5 files.
                 The HDF5 file contains by default two groups Data and Metadata where
                 you can save any `dataOut` attribute specified by `dataList` and `metadataList`
                 parameters, data attributes are normaly time dependent where the metadata
                 are not.
                 It is possible to customize the structure of the HDF5 file with the
                 optional description parameter see the examples.
                 Parameters:
                 -----------
                 path : str
                     Path where files will be saved.
                 blocksPerFile : int
                     Number of blocks per file
                 metadataList : list
                     List of the dataOut attributes that will be saved as metadata
                 dataList : int
                     List of the dataOut attributes that will be saved as data
                 setType : bool
                     If True the name of the files corresponds to the timestamp of the data
                 description : dict, optional
                     Dictionary with the desired description of the HDF5 file
                 Examples
                 --------
                 desc = {
                     'data_output': {'winds': ['z', 'w', 'v']},
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'data_output': ['z', 'w', 'v'],
                     'utctime': 'timestamps',
                     'heightList': 'heights'
                 }
                 desc = {
                     'Data': {
                         'data_output': 'winds',
                         'utctime': 'timestamps'
                     },
                     'Metadata': {
                         'heightList': 'heights'
                     }
                 }
                 writer = proc_unit.addOperation(name='HDFWriter')
                 writer.addParameter(name='path', value='/path/to/file')
                 writer.addParameter(name='blocksPerFile', value='32')
                 writer.addParameter(name='metadataList', value='heightList,timeZone')
                 writer.addParameter(name='dataList',value='data_output,utctime')
                 # writer.addParameter(name='description',value=json.dumps(desc))
                 """
                 ext = ".hdf5"
                 optchar = "D"
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 firsttime = True
                 # Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 # Data Arrays
                 dataList = None
                 metadataList = None
                 currentDay = None
                 lastTime = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
-                def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None):
+                def set_kwargs(self, **kwargs):
+                    for key, value in kwargs.items():
+                        setattr(self, key, value)
+                def set_kwargs_obj(self, obj, **kwargs):
+                    for key, value in kwargs.items():
+                        setattr(obj, key, value)
+                def setup(self, path=None, blocksPerFile=10, metadataList=None, dataList=None, setType=None, description=None, **kwargs):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataList = [s.strip() for s in dataList]
                     self.setType = setType
                     self.description = description
+                    self.set_kwargs(**kwargs)
                     if self.metadataList is None:
                         self.metadataList = self.dataOut.metadata_list
                     tableList = []
                     dsList = []
                     for i in range(len(self.dataList)):
                         dsDict = {}
                         if hasattr(self.dataOut, self.dataList[i]):
                             dataAux = getattr(self.dataOut, self.dataList[i])
                             dsDict['variable'] = self.dataList[i]
                         else:
                             log.warning('Attribute {} not found in dataOut', self.name)
                             continue
                         if dataAux is None:
                             continue
-                        elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
+                        elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                             dsDict['dtype'] = dataAux.dtype
                         dsList.append(dsDict)
                     self.dsList = dsList
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     timeTuple = time.localtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3 * 60 * 60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
-                        dataList=[], setType=None, description={}):
+                        dataList=[], setType=None, description={}, **kwargs):
                     self.dataOut = dataOut
+                    self.set_kwargs_obj(self.dataOut, **kwargs)
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
-                                   setType=setType, description=description)
+                                   setType=setType, description=description, **kwargs)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     timeTuple = time.localtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len(filesList) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11])  # inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1  # inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1  # inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext)
                     else:
                         setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext)
                     self.filename = os.path.join(path, subfolder, file)
                     # Setting HDF5 File
                     self.fp = h5py.File(self.filename, 'w')
                     # write metadata
                     self.writeMetadata(self.fp)
                     # Write data
                     self.writeData(self.fp)
                 def getLabel(self, name, x=None):
                     if x is None:
                         if 'Data' in self.description:
                             data = self.description['Data']
                             if 'Metadata' in self.description:
                                 data.update(self.description['Metadata'])
                         else:
                             data = self.description
                         if name in data:
                             if isinstance(data[name], str):
                                 return data[name]
                             elif isinstance(data[name], list):
                                 return None
                             elif isinstance(data[name], dict):
                                 for key, value in data[name].items():
                                     return key
                         return name
                     else:
-                        if 'Metadata' in self.description:
+                        if 'Data' in self.description:
-                            meta = self.description['Metadata']
+                            data = self.description['Data']
+                            if 'Metadata' in self.description:
+                                data.update(self.description['Metadata'])
                         else:
-                            meta = self.description
+                            data = self.description
-                        if name in meta:
+                        if name in data:
-                            if isinstance(meta[name], list):
+                            if isinstance(data[name], list):
-                                return meta[name][x]
+                                return data[name][x]
-                            elif isinstance(meta[name], dict):
+                            elif isinstance(data[name], dict):
-                                for key, value in meta[name].items():
+                                for key, value in data[name].items():
                                     return value[x]
                         if 'cspc' in name:
                             return 'pair{:02d}'.format(x)
                         else:
                             return 'channel{:02d}'.format(x)
                 def writeMetadata(self, fp):
                     if self.description:
                         if 'Metadata' in self.description:
                             grp = fp.create_group('Metadata')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Metadata')
                     for i in range(len(self.metadataList)):
                         if not hasattr(self.dataOut, self.metadataList[i]):
                             log.warning('Metadata: `{}` not found'.format(self.metadataList[i]), self.name)
                             continue
                         value = getattr(self.dataOut, self.metadataList[i])
                         if isinstance(value, bool):
                             if value is True:
                                 value = 1
                             else:
                                 value = 0
                         grp.create_dataset(self.getLabel(self.metadataList[i]), data=value)
                     return
                 def writeData(self, fp):
                     if self.description:
                         if 'Data' in self.description:
                             grp = fp.create_group('Data')
                         else:
                             grp = fp
                     else:
                         grp = fp.create_group('Data')
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 self.getLabel(dsInfo['variable']),
                                 (self.blocksPerFile,),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                         else:
                             label = self.getLabel(dsInfo['variable'])
                             if label is not None:
                                 sgrp = grp.create_group(label)
                             else:
                                 sgrp = grp
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     self.getLabel(dsInfo['variable'], i),
                                     (self.blocksPerFile,) + dsInfo['shape'][1:],
                                     chunks=True,
                                     dtype=dsInfo['dtype'])
                                 dtsets.append(ds)
                                 data.append((dsInfo['variable'], i))
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                     for i, ds in enumerate(self.ds):
                         attr, ch = self.data[i]
                         if ch == -1:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)
                         else:
                             ds[self.blockIndex] = getattr(self.dataOut, attr)[ch]
                     self.fp.flush()
                     self.blockIndex += 1
                     log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
                     return
                 def closeFile(self):
                     if self.blockIndex != self.blocksPerFile:
                         for ds in self.ds:
                             ds.resize(self.blockIndex, axis=0)
                     if self.fp:
                         self.fp.flush()
                         self.fp.close()
                 def close(self):
                     self.closeFile()
             @MPDecorator
             class ASCIIWriter(Operation):
                 """Operation to write data in ascii files.
                 Parameters:
                 -----------
                 path : str
                     Path where files will be saved.
                 blocksPerFile : int
                     Number of blocks per file
                 metadataList : list
                     List of the dataOut attributes that will be saved as metadata
                 dataDict : dict
                     Dictionary with the varaibles to be saved
                 setType : bool
                     If True the name of the files corresponds to the timestamp of the data
                 Examples
                 --------
                 data = {
                     'data_output': ['z', 'w', 'v'],
                     'utctime': 'time',
                     'heightList': 'height'
                 }
                 writer = proc_unit.addOperation(name='ASCIIWriter')
                 writer.addParameter(name='path', value='/path/to/file')
                 writer.addParameter(name='blocksPerFile', value='32')
                 writer.addParameter(name='dataDict',value=json.dumps(data))
                 """
                 ext = ".txt"
                 optchar = "D"
                 filename = None
                 path = None
                 setFile = None
                 fp = None
                 firsttime = True
                 # Configurations
                 blocksPerFile = None
                 blockIndex = None
                 dataOut = None
                 # Data Arrays
                 dataDict = None
                 metadataList = None
                 currentDay = None
                 lastTime = None
                 localtime = True
                 def __init__(self):
                     Operation.__init__(self)
                     return
                 def setup(self, path=None, blocksPerFile=10, metadataList=None, dataDict=None, setType=None, localtime=True):
                     self.path = path
                     self.blocksPerFile = blocksPerFile
                     self.metadataList = metadataList
                     self.dataDict = dataDict
                     self.setType = setType
                     self.localtime = localtime
                     if self.metadataList is None:
                         self.metadataList = self.dataOut.metadata_list
                     dsList = []
                     for key, value in self.dataDict.items():
                         dsDict = {}
                         if hasattr(self.dataOut, key):
                             dataAux = getattr(self.dataOut, key)
                             dsDict['variable'] = key
                         else:
                             log.warning('Attribute {} not found in dataOut', self.name)
                             continue
                         if dataAux is None:
                             continue
-                        elif isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
+                        elif isinstance(dataAux, (int, float, numpy.integer, numpy.float32)):
                             dsDict['nDim'] = 0
                         else:
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                             dsDict['dtype'] = dataAux.dtype
                         dsList.append(dsDict)
                     self.dsList = dsList
                     self.currentDay = self.dataOut.datatime.date()
                 def timeFlag(self):
                     currentTime = self.dataOut.utctime
                     if self.localtime:
                         timeTuple = time.localtime(currentTime)
                     else:
                         timeTuple = time.gmtime(currentTime)
                     dataDay = timeTuple.tm_yday
                     if self.lastTime is None:
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     # Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     if dataDay != self.currentDay:
                         self.currentDay = dataDay
                         return True
                     elif timeDiff > 3 * 60 * 60:
                         self.lastTime = currentTime
                         return True
                     else:
                         self.lastTime = currentTime
                         return False
                 def run(self, dataOut, path, blocksPerFile=10, metadataList=None,
                         dataDict={}, setType=None, localtime=True):
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataDict=dataDict,
                                    setType=setType, localtime=localtime)
                         self.isConfig = True
                         self.setNextFile()
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     if self.localtime:
                         timeTuple = time.localtime(self.dataOut.utctime)
                     else:
                         timeTuple = time.gmtime(self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
                     fullpath = os.path.join(path, subfolder)
                     if os.path.exists(fullpath):
                         filesList = os.listdir(fullpath)
                         filesList = [k for k in filesList if k.startswith(self.optchar)]
                         if len(filesList) > 0:
                             filesList = sorted(filesList, key=str.lower)
                             filen = filesList[-1]
                             # el filename debera tener el siguiente formato
                             # 0 1234 567 89A BCDE (hex)
                             # x YYYY DDD SSS .ext
                             if isNumber(filen[8:11]):
                                 setFile = int(filen[8:11])  # inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 setFile = -1
                         else:
                             setFile = -1  # inicializo mi contador de seteo
                     else:
                         os.makedirs(fullpath)
                         setFile = -1  # inicializo mi contador de seteo
                     if self.setType is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext)
                     else:
                         setFile = timeTuple.tm_hour * 60 + timeTuple.tm_min
                         file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
                                                        timeTuple.tm_year,
                                                        timeTuple.tm_yday,
                                                        setFile,
                                                        ext)
                     self.filename = os.path.join(path, subfolder, file)
                     # Setting HDF5 File
                     self.fp = open(self.filename, 'w')
                     # write metadata
                     self.writeMetadata(self.fp)
                     # Write data
                     self.writeData(self.fp)
                 def writeMetadata(self, fp):
                     line = ''
                     for d in self.dsList:
                         par = self.dataDict[d['variable']]
                         if isinstance(par, (list,tuple)):
                             for p in par:
                                 line += '{:>16}'.format(p)
                         else:
                             line += '{:>16}'.format(par)
                     line += '\n'
                     fp.write(line)
                 def writeData(self, fp):
                     log.log('Creating file: {}'.format(self.filename), self.name)
                     self.firsttime = True
                     self.blockIndex = 0
                     return
                 def putData(self):
                     if (self.blockIndex == self.blocksPerFile) or self.timeFlag():
                         self.closeFile()
                         self.setNextFile()
                     line = ''
                     for j in range(len(self.dataOut.heightList)):
                         for ds in self.dsList:
                             par = self.dataDict[ds['variable']]
                             if ds['nDim'] == 2:
                                 for i in range(len(par)):
                                     line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[i][j])
                             elif ds['nDim'] == 1:
                                 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable'])[j])
                             else:
                                 line += '{:>16}'.format('%8.2f' % getattr(self.dataOut, ds['variable']))
                         line += '\n'
                     self.fp.write(line)
                     self.blockIndex += 1
                     log.log('Block No. {}/{}'.format(self.blockIndex, self.blocksPerFile), self.name)
                     return
                 def closeFile(self):
                     if self.fp:
                         self.fp.close()
                 def close(self):
                     self.closeFile()

schainpy/model/io/jroIO_simulator.py +3 -3

             import numpy, math, random, time
             #---------------1  Heredamos JRODatareader
             from schainpy.model.io.jroIO_base import *
             #---------------2 Heredamos las propiedades de ProcessingUnit
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             #---------------3 Importaremos las clases BascicHeader, SystemHeader, RadarControlHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             #---------------4 Importaremos el objeto Voltge
             from schainpy.model.data.jrodata import Voltage
             class SimulatorReader(JRODataReader, ProcessingUnit):
                 incIntFactor = 1
                 nFFTPoints = 0
                 FixPP_IncInt = 1
                 FixRCP_IPP = 1000
                 FixPP_CohInt = 1
                 Tau_0 = 250
                 AcqH0_0 = 70
                 H0 = AcqH0_0
                 AcqDH_0 = 1.25
                 DH0 = AcqDH_0
                 Bauds = 32
                 BaudWidth = None
                 FixRCP_TXA = 40
                 FixRCP_TXB = 70
                 fAngle = 2.0 * math.pi * (1 / 16)
                 DC_level = 500
                 stdev = 8
                 Num_Codes = 2
                 # code0                          = numpy.array([1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1])
                 # code1                          = numpy.array([1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,1,0,1,1,1,0,0,0,1,0])
                 # Dyn_snCode                     = numpy.array([Num_Codes,Bauds])
                 Dyn_snCode = None
                 Samples = 200
                 channels = 2
                 pulses = None
                 Reference = None
                 pulse_size = None
                 prof_gen = None
                 Fdoppler = 100
                 Hdoppler = 36
                 Adoppler = 300
                 frequency = 9345
                 nTotalReadFiles = 1000
                 def __init__(self):
                     """
                     Inicializador de la clases SimulatorReader para
                     generar datos de voltage simulados.
                     Input:
                         dataOut: Objeto de la clase Voltage.
                         Este Objeto sera utilizado apra almacenar
                         un perfil de datos  cada vez qe se haga
                         un requerimiento (getData)
                     """
                     ProcessingUnit.__init__(self)
                     print(" [ START ] init - Metodo Simulator Reader")
                     self.isConfig = False
                     self.basicHeaderObj = BasicHeader(LOCALTIME)
                     self.systemHeaderObj = SystemHeader()
                     self.radarControllerHeaderObj = RadarControllerHeader()
                     self.processingHeaderObj = ProcessingHeader()
                     self.profileIndex = 2 ** 32 - 1
                     self.dataOut = Voltage()
                     # code0                          = numpy.array([1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,1,1,1,0,1,1,0,1,0,0,0,1,1,1,0,1])
                     code0 = numpy.array([1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, 1, 1, 1, -1, 1, 1, -1, 1, -1, -1, -1, 1, 1, 1, -1, 1])
                     # code1                          = numpy.array([1,1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,1,0,1,1,1,0,0,0,1,0])
                     code1 = numpy.array([1, 1, 1, -1, 1, 1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, -1, -1, 1, -1, -1, 1, -1, 1, 1, 1, -1, -1, -1, 1, -1])
                     # self.Dyn_snCode                = numpy.array([code0,code1])
                     self.Dyn_snCode = None
                 def set_kwargs(self, **kwargs):
                     for key, value in kwargs.items():
                         setattr(self, key, value)
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
                         if self.nReadBlocks > 0:
                             tmp = self.dataOut.utctime
                             tmp_utc = int(self.dataOut.utctime)
                             tmp_milisecond = int((tmp - tmp_utc) * 1000)
                             self.basicHeaderObj.utc = tmp_utc
                             self.basicHeaderObj.miliSecond = tmp_milisecond
                         return 1
                     return 0
                 def setNextFile(self):
                     """Set the next file to be readed open it and parse de file header"""
                     if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
                         self.nReadFiles = self.nReadFiles + 1
                         if self.nReadFiles > self.nTotalReadFiles:
                             self.flagNoMoreFiles = 1
                             raise schainpy.admin.SchainWarning('No more files to read')
                         print('------------------- [Opening file] ------------------------------', self.nReadFiles)
                         self.nReadBlocks = 0
                     # if self.nReadBlocks==0:
                     #    self.readFirstHeader()
                 def __setNewBlock(self):
                     self.setNextFile()
                     if self.flagIsNewFile:
                         return 1
                 def readNextBlock(self):
                     while True:
                         self.__setNewBlock()
                         if not(self.readBlock()):
                             return 0
                         self.getBasicHeader()
                         break
                     if self.verbose:
                         print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
                                                                   self.processingHeaderObj.dataBlocksPerFile,
                                                                   self.dataOut.datatime.ctime()))
                     return 1
                 def getFirstHeader(self):
                     self.getBasicHeader()
                     self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
                     self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
                     self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
                     self.dataOut.dtype = self.dtype
                     self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
                     self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
                     self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
                     self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
                     # asumo q la data no esta decodificada
                     self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
                     # asumo q la data no esta sin flip
                     self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
                     self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
                     self.dataOut.frequency = self.frequency
                 def getBasicHeader(self):
                     self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
 . + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     self.dataOut.timeZone = self.basicHeaderObj.timeZone
                     self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
                     self.dataOut.errorCount = self.basicHeaderObj.errorCount
                     self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
                     self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
                 def readFirstHeader(self):
                     datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
                                                PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
                     if datatype == 0:
                         datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
                     elif datatype == 1:
                         datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
                     elif datatype == 2:
                         datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
                     elif datatype == 3:
                         datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
                     elif datatype == 4:
                         datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
                     elif datatype == 5:
                         datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
                     else:
                         raise ValueError('Data type was not defined')
                     self.dtype = datatype_str
                 def set_RCH(self, expType=2, nTx=1, ipp=None, txA=0, txB=0,
                              nWindows=None, nHeights=None, firstHeight=None, deltaHeight=None,
                              numTaus=0, line6Function=0, line5Function=0, fClock=None,
                              prePulseBefore=0, prePulseAfter=0,
                              codeType=0, nCode=0, nBaud=0, code=None,
                              flip1=0, flip2=0, Taus=0):
                     self.radarControllerHeaderObj.expType = expType
                     self.radarControllerHeaderObj.nTx = nTx
                     self.radarControllerHeaderObj.ipp = float(ipp)
                     self.radarControllerHeaderObj.txA = float(txA)
                     self.radarControllerHeaderObj.txB = float(txB)
                     self.radarControllerHeaderObj.rangeIpp = b'A\n'  # ipp
                     self.radarControllerHeaderObj.rangeTxA = b''
                     self.radarControllerHeaderObj.rangeTxB = b''
                     self.radarControllerHeaderObj.nHeights = int(nHeights)
                     self.radarControllerHeaderObj.firstHeight = numpy.array([firstHeight])
                     self.radarControllerHeaderObj.deltaHeight = numpy.array([deltaHeight])
                     self.radarControllerHeaderObj.samplesWin = numpy.array([nHeights])
                     self.radarControllerHeaderObj.nWindows = nWindows
                     self.radarControllerHeaderObj.numTaus = numTaus
                     self.radarControllerHeaderObj.codeType = codeType
                     self.radarControllerHeaderObj.line6Function = line6Function
                     self.radarControllerHeaderObj.line5Function = line5Function
                     # self.radarControllerHeaderObj.fClock        = fClock
                     self.radarControllerHeaderObj.prePulseBefore = prePulseBefore
                     self.radarControllerHeaderObj.prePulseAfter = prePulseAfter
                     self.radarControllerHeaderObj.flip1 = flip1
                     self.radarControllerHeaderObj.flip2 = flip2
                     self.radarControllerHeaderObj.code_size = 0
                     if self.radarControllerHeaderObj.codeType != 0:
                         self.radarControllerHeaderObj.nCode = nCode
                         self.radarControllerHeaderObj.nBaud = nBaud
                         self.radarControllerHeaderObj.code = code
                         self.radarControllerHeaderObj.code_size = int(numpy.ceil(nBaud / 32.)) * nCode * 4
                     if fClock is None and deltaHeight is not None:
                         self.fClock = 0.15 / (deltaHeight * 1e-6)
                         self.radarControllerHeaderObj.fClock = self.fClock
                     if numTaus == 0:
                         self.radarControllerHeaderObj.Taus = numpy.array(0, '<f4')
                     else:
                         self.radarControllerHeaderObj.Taus = numpy.array(Taus, '<f4')
                 def set_PH(self, dtype=0, blockSize=0, profilesPerBlock=0,
                               dataBlocksPerFile=0, nWindows=0, processFlags=0, nCohInt=0,
                               nIncohInt=0, totalSpectra=0, nHeights=0, firstHeight=0,
                               deltaHeight=0, samplesWin=0, spectraComb=0, nCode=0,
                               code=0, nBaud=None, shif_fft=False, flag_dc=False,
                               flag_cspc=False, flag_decode=False, flag_deflip=False):
                     self.processingHeaderObj.dtype = dtype
                     self.processingHeaderObj.profilesPerBlock = profilesPerBlock
                     self.processingHeaderObj.dataBlocksPerFile = dataBlocksPerFile
                     self.processingHeaderObj.nWindows = nWindows
                     self.processingHeaderObj.processFlags = processFlags
                     self.processingHeaderObj.nCohInt = nCohInt
                     self.processingHeaderObj.nIncohInt = nIncohInt
                     self.processingHeaderObj.totalSpectra = totalSpectra
                     self.processingHeaderObj.nHeights = int(nHeights)
                     self.processingHeaderObj.firstHeight = firstHeight  # numpy.array([firstHeight])#firstHeight
                     self.processingHeaderObj.deltaHeight = deltaHeight  # numpy.array([deltaHeight])#deltaHeight
                     self.processingHeaderObj.samplesWin = nHeights  # numpy.array([nHeights])#nHeights
                 def set_BH(self, utc=0, miliSecond=0, timeZone=0):
                     self.basicHeaderObj.utc = utc
                     self.basicHeaderObj.miliSecond = miliSecond
                     self.basicHeaderObj.timeZone = timeZone
                 def set_SH(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=32):
                     # self.systemHeaderObj.size           = size
                     self.systemHeaderObj.nSamples = nSamples
                     self.systemHeaderObj.nProfiles = nProfiles
                     self.systemHeaderObj.nChannels = nChannels
                     self.systemHeaderObj.adcResolution = adcResolution
                     self.systemHeaderObj.pciDioBusWidth = pciDioBusWidth
                 def init_acquisition(self):
                     if self.nFFTPoints != 0:
                         self.incIntFactor = m_nProfilesperBlock / self.nFFTPoints
                         if (self.FixPP_IncInt > self.incIntFactor):
                             self.incIntFactor = self.FixPP_IncInt / self.incIntFactor
                         elif(self.FixPP_IncInt < self.incIntFactor):
                             print("False alert...")
                     ProfilesperBlock = self.processingHeaderObj.profilesPerBlock
                     self.timeperblock = int(((self.FixRCP_IPP
                                     * ProfilesperBlock
                                     * self.FixPP_CohInt
                                     * self.incIntFactor)
                                    / 150.0)
                                   * 0.9
                                   + 0.5)
                     # para cada canal
                     self.profiles = ProfilesperBlock * self.FixPP_CohInt
                     self.profiles = ProfilesperBlock
                     self.Reference = int((self.Tau_0 - self.AcqH0_0) / (self.AcqDH_0) + 0.5)
                     self.BaudWidth = int((self.FixRCP_TXA / self.AcqDH_0) / self.Bauds + 0.5)
                     if (self.BaudWidth == 0):
                         self.BaudWidth = 1
                 def init_pulse(self, Num_Codes=Num_Codes, Bauds=Bauds, BaudWidth=BaudWidth, Dyn_snCode=Dyn_snCode):
                     Num_Codes = Num_Codes
                     Bauds = Bauds
                     BaudWidth = BaudWidth
                     Dyn_snCode = Dyn_snCode
                     if Dyn_snCode:
                         print("EXISTE")
                     else:
                         print("No existe")
                     if Dyn_snCode:  # if Bauds:
                         pulses = list(range(0, Num_Codes))
                         num_codes = Num_Codes
                         for i in range(num_codes):
                             pulse_size = Bauds * BaudWidth
                             pulses[i] = numpy.zeros(pulse_size)
                             for j in range(Bauds):
                                 for k in range(BaudWidth):
                                     pulses[i][j * BaudWidth + k] = int(Dyn_snCode[i][j] * 600)
                     else:
                         print("sin code")
                         pulses = list(range(1))
                         if self.AcqDH_0 > 0.149:
                             pulse_size = int(self.FixRCP_TXB / 0.15 + 0.5)
                         else:
                             pulse_size = int((self.FixRCP_TXB / self.AcqDH_0) + 0.5)  # 0.0375
                         pulses[0] = numpy.ones(pulse_size)
                         pulses = 600 * pulses[0]
                     return pulses, pulse_size
                 def jro_GenerateBlockOfData(self, Samples=Samples, DC_level=DC_level, stdev=stdev,
                                             Reference=Reference, pulses=pulses,
                                             Num_Codes=Num_Codes, pulse_size=pulse_size,
                                             prof_gen=prof_gen, H0=H0, DH0=DH0,
                                             Adoppler=Adoppler, Fdoppler=Fdoppler, Hdoppler=Hdoppler):
                     Samples = Samples
                     DC_level = DC_level
                     stdev = stdev
                     m_nR = Reference
                     pulses = pulses
                     num_codes = Num_Codes
                     ps = pulse_size
                     prof_gen = prof_gen
                     channels = self.channels
                     H0 = H0
                     DH0 = DH0
                     ippSec = self.radarControllerHeaderObj.ippSeconds
                     Fdoppler = self.Fdoppler
                     Hdoppler = self.Hdoppler
                     Adoppler = self.Adoppler
-                    self.datablock = numpy.zeros([channels, prof_gen, Samples], dtype=numpy.complex64)
+                    self.datablock = numpy.zeros([channels, prof_gen, Samples], dtype=complex)
                     for i in range(channels):
                         for k in range(prof_gen):
                             #-----------------------NOISE---------------
                             Noise_r = numpy.random.normal(DC_level, stdev, Samples)
                             Noise_i = numpy.random.normal(DC_level, stdev, Samples)
                             Noise = numpy.zeros(Samples, dtype=complex)
                             Noise.real = Noise_r
                             Noise.imag = Noise_i
                             #-----------------------PULSOS--------------
                             Pulso = numpy.zeros(pulse_size, dtype=complex)
                             Pulso.real = pulses[k % num_codes]
                             Pulso.imag = pulses[k % num_codes]
                             #--------------------- PULSES+NOISE----------
                             InBuffer = numpy.zeros(Samples, dtype=complex)
                             InBuffer[m_nR:m_nR + ps] = Pulso
                             InBuffer = InBuffer + Noise
                             #--------------------- ANGLE -------------------------------
                             InBuffer.real[m_nR:m_nR + ps] = InBuffer.real[m_nR:m_nR + ps] * (math.cos(self.fAngle) * 5)
                             InBuffer.imag[m_nR:m_nR + ps] = InBuffer.imag[m_nR:m_nR + ps] * (math.sin(self.fAngle) * 5)
                             InBuffer = InBuffer
                             self.datablock[i][k] = InBuffer
                     #----------------DOPPLER SIGNAL...............................................
                     time_vec = numpy.linspace(0, (prof_gen - 1) * ippSec, int(prof_gen)) + self.nReadBlocks * ippSec * prof_gen + (self.nReadFiles - 1) * ippSec * prof_gen
                     fd = Fdoppler  # +(600.0/120)*self.nReadBlocks
-                    d_signal = Adoppler * numpy.array(numpy.exp(1.0j * 2.0 * math.pi * fd * time_vec), dtype=numpy.complex64)
+                    d_signal = Adoppler * numpy.array(numpy.exp(1.0j * 2.0 * math.pi * fd * time_vec), dtype=complex)
                     #-------------Senal con ancho espectral--------------------
                     if prof_gen % 2 == 0:
                         min = int(prof_gen / 2.0 - 1.0)
                         max = int(prof_gen / 2.0)
                     else:
                         min = int(prof_gen / 2.0)
                         max = int(prof_gen / 2.0)
                     specw_sig = numpy.linspace(-min, max, prof_gen)
                     w = 4
                     A = 20
                     specw_sig = specw_sig / w
                     specw_sig = numpy.sinc(specw_sig)
-                    specw_sig = A * numpy.array(specw_sig, dtype=numpy.complex64)
+                    specw_sig = A * numpy.array(specw_sig, dtype=complex)
                     #------------------ DATABLOCK + DOPPLER--------------------
                     HD = int(Hdoppler / self.AcqDH_0)
                     for  i in range(12):
                         self.datablock[0, :, HD + i] = self.datablock[0, :, HD + i] + d_signal  # RESULT
                     #------------------ DATABLOCK + DOPPLER*Sinc(x)--------------------
                     HD = int(Hdoppler / self.AcqDH_0)
                     HD = int(HD / 2)
                     for  i in range(12):
                         self.datablock[0, :, HD + i] = self.datablock[0, :, HD + i] + specw_sig * d_signal  # RESULT
                 def readBlock(self):
                     self.jro_GenerateBlockOfData(Samples=self.samples, DC_level=self.DC_level,
                                                  stdev=self.stdev, Reference=self.Reference,
                                                  pulses=self.pulses, Num_Codes=self.Num_Codes,
                                                  pulse_size=self.pulse_size, prof_gen=self.profiles,
                                                  H0=self.H0, DH0=self.DH0)
                     self.profileIndex = 0
                     self.flagIsNewFile = 0
                     self.flagIsNewBlock = 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNodata = True
                         return 0
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if self.__hasNotDataInBuffer():  # aqui es verdad
                         if not(self.readNextBlock()):  # return 1 y por eso el if not salta  a getBasic Header
                             return 0
                         self.getFirstHeader()  # atributo
                     if not self.getByBlock:
                         self.dataOut.flagDataAsBlock = False
                         self.dataOut.data = self.datablock[:, self.profileIndex, :]
                         self.dataOut.profileIndex = self.profileIndex
                         self.profileIndex += 1
                     else:
                         pass
                     self.dataOut.flagNoData = False
                     self.getBasicHeader()
                     self.dataOut.realtime = self.online
                     return self.dataOut.data
                 def setup(self, frequency=49.92e6, incIntFactor=1, nFFTPoints=0, FixPP_IncInt=1, FixRCP_IPP=1000,
                                FixPP_CohInt=1, Tau_0=250, AcqH0_0=70 , AcqDH_0=1.25, Bauds=32,
                                FixRCP_TXA=40, FixRCP_TXB=50, fAngle=2.0 * math.pi * (1 / 16), DC_level=50,
                                stdev=8, Num_Codes=1 , Dyn_snCode=None, samples=200,
                                channels=2, Fdoppler=20, Hdoppler=36, Adoppler=500,
                                profilesPerBlock=300, dataBlocksPerFile=120, nTotalReadFiles=10000,
                                **kwargs):
                     self.set_kwargs(**kwargs)
                     self.nReadBlocks = 0
                     self.nReadFiles = 1
                     print('------------------- [Opening file: ] ------------------------------', self.nReadFiles)
                     tmp = time.time()
                     tmp_utc = int(tmp)
                     tmp_milisecond = int((tmp - tmp_utc) * 1000)
                     print(" SETUP -basicHeaderObj.utc", datetime.datetime.utcfromtimestamp(tmp))
                     if Dyn_snCode is None:
                         Num_Codes = 1
                         Bauds = 1
                     self.set_BH(utc=tmp_utc, miliSecond=tmp_milisecond, timeZone=300)
                     self.set_RCH(expType=0, nTx=150, ipp=FixRCP_IPP, txA=FixRCP_TXA, txB=FixRCP_TXB,
                              nWindows=1 , nHeights=samples, firstHeight=AcqH0_0, deltaHeight=AcqDH_0,
                              numTaus=1, line6Function=0, line5Function=0, fClock=None,
                              prePulseBefore=0, prePulseAfter=0,
                              codeType=0, nCode=Num_Codes, nBaud=32, code=Dyn_snCode,
                              flip1=0, flip2=0, Taus=Tau_0)
                     self.set_PH(dtype=0, blockSize=0, profilesPerBlock=profilesPerBlock,
                                   dataBlocksPerFile=dataBlocksPerFile, nWindows=1, processFlags=numpy.array([1024]), nCohInt=1,
                                   nIncohInt=1, totalSpectra=0, nHeights=samples, firstHeight=AcqH0_0,
                                   deltaHeight=AcqDH_0, samplesWin=samples, spectraComb=0, nCode=0,
                                   code=0, nBaud=None, shif_fft=False, flag_dc=False,
                                   flag_cspc=False, flag_decode=False, flag_deflip=False)
                     self.set_SH(nSamples=samples, nProfiles=profilesPerBlock, nChannels=channels)
                     self.readFirstHeader()
                     self.frequency = frequency
                     self.incIntFactor = incIntFactor
                     self.nFFTPoints = nFFTPoints
                     self.FixPP_IncInt = FixPP_IncInt
                     self.FixRCP_IPP = FixRCP_IPP
                     self.FixPP_CohInt = FixPP_CohInt
                     self.Tau_0 = Tau_0
                     self.AcqH0_0 = AcqH0_0
                     self.H0 = AcqH0_0
                     self.AcqDH_0 = AcqDH_0
                     self.DH0 = AcqDH_0
                     self.Bauds = Bauds
                     self.FixRCP_TXA = FixRCP_TXA
                     self.FixRCP_TXB = FixRCP_TXB
                     self.fAngle = fAngle
                     self.DC_level = DC_level
                     self.stdev = stdev
                     self.Num_Codes = Num_Codes
                     self.Dyn_snCode = Dyn_snCode
                     self.samples = samples
                     self.channels = channels
                     self.profiles = None
                     self.m_nReference = None
                     self.Baudwidth = None
                     self.Fdoppler = Fdoppler
                     self.Hdoppler = Hdoppler
                     self.Adoppler = Adoppler
                     self.nTotalReadFiles = int(nTotalReadFiles)
                     print("IPP    ", self.FixRCP_IPP)
                     print("Tau_0  ", self.Tau_0)
                     print("AcqH0_0", self.AcqH0_0)
                     print("samples,window ", self.samples)
                     print("AcqDH_0", AcqDH_0)
                     print("FixRCP_TXA", self.FixRCP_TXA)
                     print("FixRCP_TXB", self.FixRCP_TXB)
                     print("Dyn_snCode", Dyn_snCode)
                     print("Fdoppler", Fdoppler)
                     print("Hdoppler", Hdoppler)
                     print("Vdopplermax", Fdoppler * (3.0e8 / self.frequency) / 2.0)
                     print("nTotalReadFiles", nTotalReadFiles)
                     self.init_acquisition()
                     self.pulses, self.pulse_size = self.init_pulse(Num_Codes=self.Num_Codes, Bauds=self.Bauds, BaudWidth=self.BaudWidth, Dyn_snCode=Dyn_snCode)
                     print(" [ END ] - SETUP metodo")
                     return
                 def run(self, **kwargs):  # metodo propio
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

schainpy/model/io/jroIO_usrp.py +3 -3

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import os
             import datetime
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             try:
                 import digital_rf_hdf5
             except:
                 pass
             class USRPReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self, **kwargs)
                     self.dataOut = Voltage()
                     self.__printInfo = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__ippKm = None
                     self.__codeType = 0
                     self.__nCode = None
                     self.__nBaud = None
                     self.__code = None
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample / self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
                     nProfiles = 1.0 / ippSeconds  # Number of profiles in one second
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
                                                                                   txA=0,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeigth,
                                                                                   deltaHeight=self.__deltaHeigth,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code=self.__code)
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=nProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = nProfiles
-                    self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float) * self.__deltaHeigth
+                    self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype=numpy.float32) * self.__deltaHeigth
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     # Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     self.dataOut.timeZone = self.__timezone / 60  # timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False  # asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False  # asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = ippSeconds
                     # Time interval between profiles
                     # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
                 def findDatafiles(self, path, startDate=None, endDate=None):
                     if not os.path.isdir(path):
                         return []
                     try:
                         digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = digitalReadObj.get_channels()
                     if not channelNameList:
                         return []
                     metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
                     sample_rate = metadata_dict['sample_rate'][0]
                     this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
                     try:
                         timezone = this_metadata_file['timezone'].value
                     except:
                         timezone = 0
                     startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0]) / sample_rate - timezone
                     startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
                     endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
                     if not startDate:
                         startDate = startDatetime.date()
                     if not endDate:
                         endDate = endDatatime.date()
                     dateList = []
                     thisDatetime = startDatetime
                     while(thisDatetime <= endDatatime):
                         thisDate = thisDatetime.date()
                         if thisDate < startDate:
                             continue
                         if thisDate > endDate:
                             break
                         dateList.append(thisDate)
                         thisDatetime += datetime.timedelta(1)
                     return dateList
                 def setup(self, path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=datetime.time(0, 0, 0),
                                 endTime=datetime.time(23, 59, 59),
                                 channelList=None,
                                 nSamples=None,
                                 ippKm=60,
                                 online=False,
                                 delay=60,
                                 buffer_size=1024,
                                 **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
                         delay
                     '''
                     if not os.path.isdir(path):
                         raise ValueError("[Reading] Directory %s does not exist" % path)
                     try:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = self.digitalReadObj.get_channels()
                     if not channelNameList:
                         raise ValueError("[Reading] Directory %s does not have any files" % path)
                     if not channelList:
                         channelList = list(range(len(channelNameList)))
                     ##########  Reading metadata ######################
                     metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
                     self.__sample_rate = metadata_dict['sample_rate'][0]
             #         self.__samples_per_file = metadata_dict['samples_per_file'][0]
                     self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate
                     this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
                     self.__frequency = None
                     try:
                         self.__frequency = this_metadata_file['center_frequencies'].value
                     except:
                         self.__frequency = this_metadata_file['fc'].value
                     if not self.__frequency:
                         raise ValueError("Center Frequency is not defined in metadata file")
                     try:
                         self.__timezone = this_metadata_file['timezone'].value
                     except:
                         self.__timezone = 0
                     self.__firstHeigth = 0
                     try:
                         codeType = this_metadata_file['codeType'].value
                     except:
                         codeType = 0
                     nCode = 1
                     nBaud = 1
-                    code = numpy.ones((nCode, nBaud), dtype=numpy.int)
+                    code = numpy.ones((nCode, nBaud), dtype=numpy.int32)
                     if codeType:
                         nCode = this_metadata_file['nCode'].value
                         nBaud = this_metadata_file['nBaud'].value
                         code = this_metadata_file['code'].value
                     if not ippKm:
                         try:
                             # seconds to km
                             ippKm = 1e6 * 0.15 * this_metadata_file['ipp'].value
                         except:
                             ippKm = None
                     ####################################################
                     startUTCSecond = None
                     endUTCSecond = None
                     if startDate:
                         startDatetime = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond = (endDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index / self.__sample_rate
                     if start_index > startUTCSecond * self.__sample_rate:
                         startUTCSecond = start_index / self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond = end_index / self.__sample_rate
                     if end_index < endUTCSecond * self.__sample_rate:
                         endUTCSecond = end_index / self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError("[Reading] nSamples or ippKm should be defined")
                         nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
                     channelBoundList = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName = channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
                     self.__delay = delay
                     self.__ippKm = ippKm
                     self.__codeType = codeType
                     self.__nCode = nCode
                     self.__nBaud = nBaud
                     self.__code = code
                     self.__datapath = path
                     self.__online = online
                     self.__channelList = channelList
                     self.__channelNameList = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples = nSamples
                     self.__samples_to_read = int(buffer_size * nSamples)
                     self.__nChannels = len(self.__channelList)
                     self.__startUTCSecond = startUTCSecond
                     self.__endUTCSecond = endUTCSecond
                     self.__timeInterval = 1.0 * self.__samples_to_read / self.__sample_rate  # Time interval
                     if online:
             #             self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read
-                    self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype=numpy.complex)
+                    self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype=complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print("[Reading] USRP Data was found from %s to %s " % (
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   ))
                     print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
                                                                        datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
                                                                        ))
                 def __reload(self):
                     if not self.__online:
                         return
             #         print
             #         print "%s not in range [%s, %s]" %(
             #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
             #                                           )
                     print("[Reading] reloading metadata ...")
                     try:
                         self.digitalReadObj.reload(complete_update=True)
                     except:
                         self.digitalReadObj.reload()
                     start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
                     if start_index > self.__startUTCSecond * self.__sample_rate:
                         self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
                     if end_index > self.__endUTCSecond * self.__sample_rate:
                         self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
                         print()
                         print("[Reading] New timerange found [%s, %s] " % (
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   ))
                         return True
                     return False
                 def __readNextBlock(self, seconds=30, volt_scale=218776):
                     '''
                     '''
                     # Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample += self.__samples_to_read
                     if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                         print("[Reading] There are no more data into selected time-range")
                         self.__reload()
                         if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                             self.__thisUnixSample -= self.__samples_to_read
                             return False
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:
                         try:
                             result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                           self.__samples_to_read,
                                                                           thisChannelName)
                         except IOError as e:
                             # read next profile
                             self.__flagDiscontinuousBlock = True
                             print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
                             break
                         if result.shape[0] != self.__samples_to_read:
                             self.__flagDiscontinuousBlock = True
                             print("[Reading] %s: Too few samples were found, just %d/%d  samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                             result.shape[0],
                                                                                             self.__samples_to_read))
                             break
                         self.__data_buffer[indexChannel, :] = result * volt_scale
                         indexChannel += 1
                         dataOk = True
                     self.__utctime = self.__thisUnixSample / self.__sample_rate
                     if not dataOk:
                         return False
                     print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                      self.__samples_to_read,
                                                                      self.__timeInterval))
                     self.__bufferIndex = 0
                     return True
                 def __isBufferEmpty(self):
                     if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
                         return False
                     return True
                 def getData(self, seconds=30, nTries=5):
                     '''
                     This method gets the data from files and put the data into the dataOut object
                     In addition, increase el the buffer counter in one.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     '''
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         self.__flagDiscontinuousBlock = False
                         while True:
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
                                 return False
                             if self.__flagDiscontinuousBlock:
                                 print('[Reading] discontinuous block found ... continue with the next block')
                                 continue
                             if not self.__online:
                                 return False
                             err_counter += 1
                             if err_counter > nTries:
                                 return False
                             print('[Reading] waiting %d seconds to read a new block' % seconds)
                             sleep(seconds)
                     self.dataOut.data = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
                     self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.dataOut.profileIndex = self.profileIndex
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex += 1
                     if self.profileIndex == self.dataOut.nProfiles:
                         self.profileIndex = 0
                     return True
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
             #         self.systemHeaderObj.printInfo()
             #         self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     print(self.profileIndex)
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                     self.getData(seconds=self.__delay)
                     return
             @MPDecorator
             class USRPWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     Operation.__init__(self, **kwargs)
                     self.dataOut = None
                 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataIn        :        Input data will also be outputa data
                     '''
                     self.dataOut = dataIn
                     self.isConfig = True
                     return
                 def run(self, dataIn, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataIn        :        object with the data
                     '''
                     if not self.isConfig:
                         self.setup(dataIn, **kwargs)
             if __name__ == '__main__':
                 readObj = USRPReader()
                 while True:
                     readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
             #         readObj.printInfo()
                     readObj.printNumberOfBlock()

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages