jroIO_digitalRF.py
793 lines
| 28.7 KiB
| text/x-python
|
PythonLexer
Jose Chavez
|
r973 | ''' | ||
| Created on Jul 3, 2014 | ||||
| @author: roj-idl71 | ||||
| ''' | ||||
|
Jose Chavez
|
r991 | # SUBCHANNELS EN VEZ DE CHANNELS | ||
| # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO | ||||
| # ACTUALIZACION DE VERSION | ||||
| # HEADERS | ||||
| # MODULO DE ESCRITURA | ||||
| # METADATA | ||||
|
Jose Chavez
|
r973 | import os | ||
| r1241 | import time | |||
|
Jose Chavez
|
r973 | import datetime | ||
| import numpy | ||||
|
Jose Chavez
|
r985 | import timeit | ||
|
Jose Chavez
|
r979 | from fractions import Fraction | ||
| r1263 | from time import time | |||
| r1266 | from time import sleep | |||
|
Jose Chavez
|
r973 | |||
| r1241 | import schainpy.admin | |||
|
Jose Chavez
|
r973 | from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader | ||
| from schainpy.model.data.jrodata import Voltage | ||||
George Yong
|
r1192 | from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator | ||
|
Jose Chavez
|
r985 | |||
|
George Yong
|
r1167 | import pickle | ||
|
Jose Chavez
|
r973 | try: | ||
| import digital_rf | ||||
| except: | ||||
Juan C. Espinoza
|
r1284 | pass | ||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1092 | |||
|
Jose Chavez
|
r973 | class DigitalRFReader(ProcessingUnit): | ||
| ''' | ||||
| classdocs | ||||
| ''' | ||||
|
George Yong
|
r1192 | def __init__(self): | ||
|
Jose Chavez
|
r973 | ''' | ||
| Constructor | ||||
| ''' | ||||
|
George Yong
|
r1192 | ProcessingUnit.__init__(self) | ||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut = Voltage() | |||
| self.__printInfo = True | ||||
|
Jose Chavez
|
r973 | self.__flagDiscontinuousBlock = False | ||
| self.__bufferIndex = 9999999 | ||||
| r1263 | self.__codeType = 0 | |||
| self.__ippKm = None | ||||
| self.__nCode = None | ||||
| self.__nBaud = None | ||||
| self.__code = None | ||||
| self.dtype = None | ||||
| self.oldAverage = None | ||||
| self.path = None | ||||
|
Jose Chavez
|
r973 | |||
|
Jose Chavez
|
r991 | def close(self): | ||
|
George Yong
|
r1167 | print('Average of writing to digital rf format is ', self.oldAverage * 1000) | ||
|
Jose Chavez
|
r991 | return | ||
|
Jose Chavez
|
r973 | def __getCurrentSecond(self): | ||
|
José Chávez
|
r1092 | return self.__thisUnixSample / self.__sample_rate | ||
|
Jose Chavez
|
r973 | |||
| thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.") | ||||
| def __setFileHeader(self): | ||||
| ''' | ||||
| In this method will be initialized every parameter of dataOut object (header, no data) | ||||
| ''' | ||||
|
José Chávez
|
r1092 | ippSeconds = 1.0 * self.__nSamples / self.__sample_rate | ||
| nProfiles = 1.0 / ippSeconds # Number of profiles in one second | ||||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1054 | try: | ||
|
José Chávez
|
r1092 | self.dataOut.radarControllerHeaderObj = RadarControllerHeader( | ||
| self.__radarControllerHeader) | ||||
|
José Chávez
|
r1054 | except: | ||
| self.dataOut.radarControllerHeaderObj = RadarControllerHeader( | ||||
|
José Chávez
|
r1092 | txA=0, | ||
| txB=0, | ||||
| nWindows=1, | ||||
| nHeights=self.__nSamples, | ||||
| firstHeight=self.__firstHeigth, | ||||
| deltaHeight=self.__deltaHeigth, | ||||
| codeType=self.__codeType, | ||||
| nCode=self.__nCode, nBaud=self.__nBaud, | ||||
| code=self.__code) | ||||
|
José Chávez
|
r1054 | try: | ||
| self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader) | ||||
| except: | ||||
| self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples, | ||||
| nProfiles=nProfiles, | ||||
|
José Chávez
|
r1092 | nChannels=len( | ||
| self.__channelList), | ||||
|
José Chávez
|
r1054 | adcResolution=14) | ||
| r1263 | self.dataOut.type = "Voltage" | |||
|
José Chávez
|
r1092 | |||
| r1263 | self.dataOut.data = None | |||
|
Jose Chavez
|
r973 | |||
|
Jose Chavez
|
r981 | self.dataOut.dtype = self.dtype | ||
|
Jose Chavez
|
r973 | |||
|
Jose Chavez
|
r981 | # self.dataOut.nChannels = 0 | ||
|
Jose Chavez
|
r973 | |||
|
Jose Chavez
|
r981 | # self.dataOut.nHeights = 0 | ||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.nProfiles = int(nProfiles) | |||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.heightList = self.__firstHeigth + \ | |||
|
José Chávez
|
r1092 | numpy.arange(self.__nSamples, dtype=numpy.float) * \ | ||
| self.__deltaHeigth | ||||
|
Jose Chavez
|
r973 | |||
|
George Yong
|
r1167 | self.dataOut.channelList = list(range(self.__num_subchannels)) | ||
|
Jose Chavez
|
r973 | |||
| r1338 | self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights | |||
|
Jose Chavez
|
r973 | |||
|
Jose Chavez
|
r981 | # self.dataOut.channelIndexList = None | ||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.flagNoData = True | |||
|
Jose Chavez
|
r982 | |||
|
Jose Chavez
|
r981 | self.dataOut.flagDataAsBlock = False | ||
| # Set to TRUE if the data is discontinuous | ||||
|
Jose Chavez
|
r973 | self.dataOut.flagDiscontinuousBlock = False | ||
| r1263 | self.dataOut.utctime = None | |||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1092 | # timezone like jroheader, difference in minutes between UTC and localtime | ||
| r1263 | self.dataOut.timeZone = self.__timezone / 60 | |||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.dstFlag = 0 | |||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.errorCount = 0 | |||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1054 | try: | ||
|
José Chávez
|
r1092 | self.dataOut.nCohInt = self.fixed_metadata_dict.get( | ||
| 'nCohInt', self.nCohInt) | ||||
|
Jose Chavez
|
r982 | |||
|
José Chávez
|
r1092 | # asumo que la data esta decodificada | ||
| self.dataOut.flagDecodeData = self.fixed_metadata_dict.get( | ||||
| 'flagDecodeData', self.flagDecodeData) | ||||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1092 | # asumo que la data esta sin flip | ||
| self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData'] | ||||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT'] | |||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime'] | |||
|
José Chávez
|
r1054 | except: | ||
| pass | ||||
|
Jose Chavez
|
r973 | |||
| self.dataOut.ippSeconds = ippSeconds | ||||
|
Jose Chavez
|
r981 | # Time interval between profiles | ||
| # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt | ||||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.frequency = self.__frequency | |||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.realtime = self.__online | |||
|
Jose Chavez
|
r973 | |||
| def findDatafiles(self, path, startDate=None, endDate=None): | ||||
| if not os.path.isdir(path): | ||||
| return [] | ||||
| try: | ||||
|
José Chávez
|
r1092 | digitalReadObj = digital_rf.DigitalRFReader( | ||
| path, load_all_metadata=True) | ||||
|
Jose Chavez
|
r973 | except: | ||
| digitalReadObj = digital_rf.DigitalRFReader(path) | ||||
| r1263 | channelNameList = digitalReadObj.get_channels() | |||
|
Jose Chavez
|
r973 | |||
| if not channelNameList: | ||||
| return [] | ||||
| r1263 | metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0]) | |||
|
Jose Chavez
|
r973 | |||
| r1263 | sample_rate = metadata_dict['sample_rate'][0] | |||
|
Jose Chavez
|
r973 | |||
| this_metadata_file = digitalReadObj.get_metadata(channelNameList[0]) | ||||
| try: | ||||
| r1263 | timezone = this_metadata_file['timezone'].value | |||
|
Jose Chavez
|
r973 | except: | ||
| r1263 | timezone = 0 | |||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1092 | startUTCSecond, endUTCSecond = digitalReadObj.get_bounds( | ||
| channelNameList[0]) / sample_rate - timezone | ||||
|
Jose Chavez
|
r973 | |||
| r1263 | startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond) | |||
| endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond) | ||||
|
Jose Chavez
|
r973 | |||
| if not startDate: | ||||
| r1263 | startDate = startDatetime.date() | |||
|
Jose Chavez
|
r973 | |||
| if not endDate: | ||||
| r1263 | endDate = endDatatime.date() | |||
|
Jose Chavez
|
r973 | |||
| r1263 | dateList = [] | |||
|
Jose Chavez
|
r973 | |||
| r1263 | thisDatetime = startDatetime | |||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1092 | while(thisDatetime <= endDatatime): | ||
|
Jose Chavez
|
r973 | |||
| r1263 | thisDate = thisDatetime.date() | |||
|
Jose Chavez
|
r973 | |||
| r1263 | if thisDate < startDate: | |||
|
Jose Chavez
|
r973 | continue | ||
| r1263 | if thisDate > endDate: | |||
|
Jose Chavez
|
r973 | break | ||
| dateList.append(thisDate) | ||||
| thisDatetime += datetime.timedelta(1) | ||||
| return dateList | ||||
|
José Chávez
|
r1092 | def setup(self, path=None, | ||
| startDate=None, | ||||
| endDate=None, | ||||
| startTime=datetime.time(0, 0, 0), | ||||
| endTime=datetime.time(23, 59, 59), | ||||
| channelList=None, | ||||
| nSamples=None, | ||||
| online=False, | ||||
| delay=60, | ||||
| buffer_size=1024, | ||||
| ippKm=None, | ||||
| nCohInt=1, | ||||
| nCode=1, | ||||
| nBaud=1, | ||||
| flagDecodeData=False, | ||||
| code=numpy.ones((1, 1), dtype=numpy.int), | ||||
| **kwargs): | ||||
|
Jose Chavez
|
r973 | ''' | ||
| In this method we should set all initial parameters. | ||||
| Inputs: | ||||
| path | ||||
| startDate | ||||
| endDate | ||||
| startTime | ||||
| r1263 | endTime | |||
|
Jose Chavez
|
r973 | set | ||
| expLabel | ||||
| ext | ||||
| online | ||||
| delay | ||||
| ''' | ||||
| r1263 | self.path = path | |||
| self.nCohInt = nCohInt | ||||
|
José Chávez
|
r1092 | self.flagDecodeData = flagDecodeData | ||
| r1263 | self.i = 0 | |||
|
Jose Chavez
|
r973 | if not os.path.isdir(path): | ||
|
George Yong
|
r1167 | raise ValueError("[Reading] Directory %s does not exist" % path) | ||
|
Jose Chavez
|
r973 | |||
| try: | ||||
|
José Chávez
|
r1092 | self.digitalReadObj = digital_rf.DigitalRFReader( | ||
| path, load_all_metadata=True) | ||||
|
Jose Chavez
|
r973 | except: | ||
| self.digitalReadObj = digital_rf.DigitalRFReader(path) | ||||
| r1263 | channelNameList = self.digitalReadObj.get_channels() | |||
|
Jose Chavez
|
r973 | |||
| if not channelNameList: | ||||
|
George Yong
|
r1167 | raise ValueError("[Reading] Directory %s does not have any files" % path) | ||
|
Jose Chavez
|
r973 | |||
| if not channelList: | ||||
|
George Yong
|
r1167 | channelList = list(range(len(channelNameList))) | ||
|
Jose Chavez
|
r973 | |||
| ########## Reading metadata ###################### | ||||
| r1263 | top_properties = self.digitalReadObj.get_properties( | |||
|
José Chávez
|
r1092 | channelNameList[channelList[0]]) | ||
|
Jose Chavez
|
r985 | |||
| r1263 | self.__num_subchannels = top_properties['num_subchannels'] | |||
| self.__sample_rate = 1.0 * \ | ||||
|
José Chávez
|
r1092 | top_properties['sample_rate_numerator'] / \ | ||
| top_properties['sample_rate_denominator'] | ||||
|
Jose Chavez
|
r981 | # self.__samples_per_file = top_properties['samples_per_file'][0] | ||
| r1263 | self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate # why 0.15? | |||
|
Jose Chavez
|
r973 | |||
| r1263 | this_metadata_file = self.digitalReadObj.get_digital_metadata( | |||
|
José Chávez
|
r1092 | channelNameList[channelList[0]]) | ||
| r1263 | metadata_bounds = this_metadata_file.get_bounds() | |||
|
José Chávez
|
r1092 | self.fixed_metadata_dict = this_metadata_file.read( | ||
| metadata_bounds[0])[metadata_bounds[0]] # GET FIRST HEADER | ||||
|
José Chávez
|
r1054 | |||
| try: | ||||
| r1263 | self.__processingHeader = self.fixed_metadata_dict['processingHeader'] | |||
|
José Chávez
|
r1054 | self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader'] | ||
| r1263 | self.__systemHeader = self.fixed_metadata_dict['systemHeader'] | |||
| self.dtype = pickle.loads(self.fixed_metadata_dict['dtype']) | ||||
|
José Chávez
|
r1054 | except: | ||
| pass | ||||
|
Jose Chavez
|
r973 | |||
| self.__frequency = None | ||||
|
Jose Chavez
|
r981 | |||
|
José Chávez
|
r1054 | self.__frequency = self.fixed_metadata_dict.get('frequency', 1) | ||
|
Jose Chavez
|
r973 | |||
Alexander Valdez
|
r1239 | self.__timezone = self.fixed_metadata_dict.get('timezone', 18000) | ||
|
Jose Chavez
|
r973 | |||
|
Jose Chavez
|
r981 | try: | ||
|
Jose Chavez
|
r982 | nSamples = self.fixed_metadata_dict['nSamples'] | ||
|
Jose Chavez
|
r981 | except: | ||
| nSamples = None | ||||
|
José Chávez
|
r1092 | |||
|
Jose Chavez
|
r973 | self.__firstHeigth = 0 | ||
| try: | ||||
| r1263 | codeType = self.__radarControllerHeader['codeType'] | |||
|
Jose Chavez
|
r973 | except: | ||
| r1263 | codeType = 0 | |||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1054 | try: | ||
| if codeType: | ||||
| nCode = self.__radarControllerHeader['nCode'] | ||||
| nBaud = self.__radarControllerHeader['nBaud'] | ||||
| r1263 | code = self.__radarControllerHeader['code'] | |||
|
José Chávez
|
r1054 | except: | ||
| pass | ||||
|
José Chávez
|
r1092 | |||
|
Jose Chavez
|
r973 | if not ippKm: | ||
| try: | ||||
|
Jose Chavez
|
r981 | # seconds to km | ||
|
Jose Chavez
|
r982 | ippKm = self.__radarControllerHeader['ipp'] | ||
|
Jose Chavez
|
r973 | except: | ||
| ippKm = None | ||||
| #################################################### | ||||
| r1263 | self.__ippKm = ippKm | |||
|
Jose Chavez
|
r973 | startUTCSecond = None | ||
| r1263 | endUTCSecond = None | |||
|
Jose Chavez
|
r973 | |||
| if startDate: | ||||
| r1263 | startDatetime = datetime.datetime.combine(startDate, startTime) | |||
|
José Chávez
|
r1092 | startUTCSecond = ( | ||
| startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone | ||||
|
Jose Chavez
|
r973 | |||
| if endDate: | ||||
| r1263 | endDatetime = datetime.datetime.combine(endDate, endTime) | |||
| endUTCSecond = (endDatetime - datetime.datetime(1970, | ||||
|
José Chávez
|
r1092 | 1, 1)).total_seconds() + self.__timezone | ||
|
Jose Chavez
|
r973 | |||
|
José Chávez
|
r1092 | start_index, end_index = self.digitalReadObj.get_bounds( | ||
| channelNameList[channelList[0]]) | ||||
|
Jose Chavez
|
r973 | |||
| if not startUTCSecond: | ||||
|
José Chávez
|
r1092 | startUTCSecond = start_index / self.__sample_rate | ||
|
Jose Chavez
|
r973 | |||
| r1263 | if start_index > startUTCSecond * self.__sample_rate: | |||
|
José Chávez
|
r1092 | startUTCSecond = start_index / self.__sample_rate | ||
|
Jose Chavez
|
r973 | |||
| if not endUTCSecond: | ||||
| r1263 | endUTCSecond = end_index / self.__sample_rate | |||
|
Jose Chavez
|
r973 | |||
| r1263 | if end_index < endUTCSecond * self.__sample_rate: | |||
| endUTCSecond = end_index / self.__sample_rate | ||||
|
Jose Chavez
|
r973 | if not nSamples: | ||
| if not ippKm: | ||||
|
George Yong
|
r1167 | raise ValueError("[Reading] nSamples or ippKm should be defined") | ||
| r1263 | nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate)) | |||
| channelBoundList = [] | ||||
|
Jose Chavez
|
r973 | channelNameListFiltered = [] | ||
| for thisIndexChannel in channelList: | ||||
| r1263 | thisChannelName = channelNameList[thisIndexChannel] | |||
|
José Chávez
|
r1092 | start_index, end_index = self.digitalReadObj.get_bounds( | ||
| thisChannelName) | ||||
|
Jose Chavez
|
r973 | channelBoundList.append((start_index, end_index)) | ||
| channelNameListFiltered.append(thisChannelName) | ||||
| self.profileIndex = 0 | ||||
| r1263 | self.i = 0 | |||
| self.__delay = delay | ||||
| self.__codeType = codeType | ||||
| self.__nCode = nCode | ||||
| self.__nBaud = nBaud | ||||
| self.__code = code | ||||
| self.__datapath = path | ||||
| self.__online = online | ||||
| self.__channelList = channelList | ||||
| self.__channelNameList = channelNameListFiltered | ||||
|
Jose Chavez
|
r973 | self.__channelBoundList = channelBoundList | ||
| r1263 | self.__nSamples = nSamples | |||
| self.__samples_to_read = int(nSamples) # FIJO: AHORA 40 | ||||
| self.__nChannels = len(self.__channelList) | ||||
|
Jose Chavez
|
r973 | |||
| r1263 | self.__startUTCSecond = startUTCSecond | |||
| self.__endUTCSecond = endUTCSecond | ||||
|
Jose Chavez
|
r973 | |||
| r1263 | self.__timeInterval = 1.0 * self.__samples_to_read / \ | |||
|
José Chávez
|
r1092 | self.__sample_rate # Time interval | ||
|
Jose Chavez
|
r973 | |||
| if online: | ||||
|
José Chávez
|
r1092 | # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read) | ||
|
Jose Chavez
|
r973 | startUTCSecond = numpy.floor(endUTCSecond) | ||
|
José Chávez
|
r1092 | # por que en el otro metodo lo primero q se hace es sumar samplestoread | ||
| r1263 | self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read | |||
|
Jose Chavez
|
r973 | |||
| r1263 | self.__data_buffer = numpy.zeros( | |||
|
José Chávez
|
r1092 | (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex) | ||
|
Jose Chavez
|
r973 | |||
| self.__setFileHeader() | ||||
| self.isConfig = True | ||||
|
George Yong
|
r1167 | print("[Reading] Digital RF Data was found from %s to %s " % ( | ||
|
José Chávez
|
r1092 | datetime.datetime.utcfromtimestamp( | ||
| self.__startUTCSecond - self.__timezone), | ||||
| datetime.datetime.utcfromtimestamp( | ||||
| self.__endUTCSecond - self.__timezone) | ||||
|
George Yong
|
r1167 | )) | ||
|
José Chávez
|
r1092 | |||
|
George Yong
|
r1167 | print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone), | ||
|
José Chávez
|
r1092 | datetime.datetime.utcfromtimestamp( | ||
| endUTCSecond - self.__timezone) | ||||
|
George Yong
|
r1167 | )) | ||
| r1263 | self.oldAverage = None | |||
| self.count = 0 | ||||
|
Jose Chavez
|
r991 | self.executionTime = 0 | ||
|
José Chávez
|
r1092 | |||
|
Jose Chavez
|
r973 | def __reload(self): | ||
| # print "%s not in range [%s, %s]" %( | ||||
| # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), | ||||
| # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone), | ||||
| # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone) | ||||
| # ) | ||||
|
George Yong
|
r1167 | print("[Reading] reloading metadata ...") | ||
|
Jose Chavez
|
r973 | |||
| try: | ||||
| self.digitalReadObj.reload(complete_update=True) | ||||
| except: | ||||
| r1263 | self.digitalReadObj = digital_rf.DigitalRFReader(self.path) | |||
|
Jose Chavez
|
r973 | |||
| r1263 | start_index, end_index = self.digitalReadObj.get_bounds( | |||
|
José Chávez
|
r1092 | self.__channelNameList[self.__channelList[0]]) | ||
|
Jose Chavez
|
r973 | |||
| r1263 | if start_index > self.__startUTCSecond * self.__sample_rate: | |||
|
José Chávez
|
r1092 | self.__startUTCSecond = 1.0 * start_index / self.__sample_rate | ||
|
Jose Chavez
|
r973 | |||
| r1263 | if end_index > self.__endUTCSecond * self.__sample_rate: | |||
|
José Chávez
|
r1092 | self.__endUTCSecond = 1.0 * end_index / self.__sample_rate | ||
|
George Yong
|
r1167 | print() | ||
| print("[Reading] New timerange found [%s, %s] " % ( | ||||
|
José Chávez
|
r1092 | datetime.datetime.utcfromtimestamp( | ||
| self.__startUTCSecond - self.__timezone), | ||||
| datetime.datetime.utcfromtimestamp( | ||||
| self.__endUTCSecond - self.__timezone) | ||||
|
George Yong
|
r1167 | )) | ||
|
Jose Chavez
|
r973 | |||
| return True | ||||
| return False | ||||
|
Jose Chavez
|
r991 | def timeit(self, toExecute): | ||
| r1263 | t0 = time.time() | |||
|
Jose Chavez
|
r991 | toExecute() | ||
| r1263 | self.executionTime = time.time() - t0 | |||
|
José Chávez
|
r1092 | if self.oldAverage is None: | ||
| self.oldAverage = self.executionTime | ||||
| r1263 | self.oldAverage = (self.executionTime + self.count * | |||
|
José Chávez
|
r1092 | self.oldAverage) / (self.count + 1.0) | ||
| r1263 | self.count = self.count + 1.0 | |||
|
Jose Chavez
|
r991 | return | ||
|
José Chávez
|
r1092 | def __readNextBlock(self, seconds=30, volt_scale=1): | ||
|
Jose Chavez
|
r973 | ''' | ||
| ''' | ||||
|
Jose Chavez
|
r981 | # Set the next data | ||
|
Jose Chavez
|
r973 | self.__flagDiscontinuousBlock = False | ||
| r1263 | self.__thisUnixSample += self.__samples_to_read | |||
|
José Chávez
|
r1092 | if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate: | ||
| r1263 | print ("[Reading] There are no more data into selected time-range") | |||
|
Jose Chavez
|
r973 | if self.__online: | ||
|
Alexander Valdez
|
r1234 | sleep(3) | ||
|
Jose Chavez
|
r973 | self.__reload() | ||
| else: | ||||
| return False | ||||
|
José Chávez
|
r1092 | if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate: | ||
|
Jose Chavez
|
r973 | return False | ||
|
José Chávez
|
r1092 | self.__thisUnixSample -= self.__samples_to_read | ||
|
Jose Chavez
|
r973 | |||
| indexChannel = 0 | ||||
| dataOk = False | ||||
| r1263 | ||||
|
José Chávez
|
r1092 | for thisChannelName in self.__channelNameList: # TODO VARIOS CHANNELS? | ||
|
Jose Chavez
|
r985 | for indexSubchannel in range(self.__num_subchannels): | ||
| try: | ||||
| r1263 | t0 = time() | |||
|
Jose Chavez
|
r985 | result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample, | ||
|
Jose Chavez
|
r991 | self.__samples_to_read, | ||
| thisChannelName, sub_channel=indexSubchannel) | ||||
| r1263 | self.executionTime = time() - t0 | |||
|
José Chávez
|
r1092 | if self.oldAverage is None: | ||
| self.oldAverage = self.executionTime | ||||
| r1263 | self.oldAverage = ( | |||
|
José Chávez
|
r1092 | self.executionTime + self.count * self.oldAverage) / (self.count + 1.0) | ||
|
Jose Chavez
|
r991 | self.count = self.count + 1.0 | ||
|
José Chávez
|
r1092 | |||
|
George Yong
|
r1167 | except IOError as e: | ||
|
José Chávez
|
r1092 | # read next profile | ||
|
Jose Chavez
|
r985 | self.__flagDiscontinuousBlock = True | ||
|
George Yong
|
r1167 | print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e) | ||
|
Jose Chavez
|
r985 | break | ||
|
Jose Chavez
|
r973 | |||
|
Jose Chavez
|
r985 | if result.shape[0] != self.__samples_to_read: | ||
| self.__flagDiscontinuousBlock = True | ||||
|
George Yong
|
r1167 | print("[Reading] %s: Too few samples were found, just %d/%d samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), | ||
|
José Chávez
|
r1092 | result.shape[0], | ||
|
George Yong
|
r1167 | self.__samples_to_read)) | ||
|
Jose Chavez
|
r985 | break | ||
| r1263 | ||||
|
José Chávez
|
r1092 | self.__data_buffer[indexSubchannel, :] = result * volt_scale | ||
| r1263 | indexChannel+=1 | |||
|
Jose Chavez
|
r973 | |||
| r1263 | dataOk = True | |||
|
José Chávez
|
r1092 | |||
| r1263 | self.__utctime = self.__thisUnixSample / self.__sample_rate | |||
|
Jose Chavez
|
r973 | |||
| if not dataOk: | ||||
| return False | ||||
|
George Yong
|
r1167 | print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), | ||
|
José Chávez
|
r1092 | self.__samples_to_read, | ||
|
George Yong
|
r1167 | self.__timeInterval)) | ||
|
Jose Chavez
|
r973 | |||
| r1263 | self.__bufferIndex = 0 | |||
|
Jose Chavez
|
r973 | |||
| return True | ||||
| def __isBufferEmpty(self): | ||||
|
José Chávez
|
r1092 | return self.__bufferIndex > self.__samples_to_read - self.__nSamples # 40960 - 40 | ||
|
Jose Chavez
|
r973 | |||
| def getData(self, seconds=30, nTries=5): | ||||
| ''' | ||||
| This method gets the data from files and put the data into the dataOut object | ||||
| In addition, increase el the buffer counter in one. | ||||
| Return: | ||||
| data : retorna un perfil de voltages (alturas * canales) copiados desde el | ||||
| buffer. Si no hay mas archivos a leer retorna None. | ||||
| Affected: | ||||
| self.dataOut | ||||
| self.profileIndex | ||||
| self.flagDiscontinuousBlock | ||||
| self.flagIsNewBlock | ||||
| ''' | ||||
| r1263 | #print("getdata") | |||
|
Jose Chavez
|
r973 | err_counter = 0 | ||
| self.dataOut.flagNoData = True | ||||
| if self.__isBufferEmpty(): | ||||
| r1263 | #print("hi") | |||
|
Jose Chavez
|
r973 | self.__flagDiscontinuousBlock = False | ||
| while True: | ||||
| r1263 | #print ("q ha pasado") | |||
|
Jose Chavez
|
r973 | if self.__readNextBlock(): | ||
| break | ||||
|
José Chávez
|
r1092 | if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate: | ||
| r1241 | raise schainpy.admin.SchainError('Error') | |||
| r1263 | return | |||
|
Jose Chavez
|
r973 | |||
| if self.__flagDiscontinuousBlock: | ||||
| r1241 | raise schainpy.admin.SchainError('discontinuous block found') | |||
|
George Yong
|
r1192 | return | ||
|
Jose Chavez
|
r973 | |||
| if not self.__online: | ||||
| r1241 | raise schainpy.admin.SchainError('Online?') | |||
| r1263 | return | |||
|
Jose Chavez
|
r973 | |||
| err_counter += 1 | ||||
| if err_counter > nTries: | ||||
| r1241 | raise schainpy.admin.SchainError('Max retrys reach') | |||
|
George Yong
|
r1192 | return | ||
|
Jose Chavez
|
r973 | |||
|
George Yong
|
r1167 | print('[Reading] waiting %d seconds to read a new block' % seconds) | ||
| r1241 | time.sleep(seconds) | |||
|
Jose Chavez
|
r973 | |||
| r1263 | self.dataOut.data = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples] | |||
| self.dataOut.utctime = ( self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate | ||||
| self.dataOut.flagNoData = False | ||||
|
Jose Chavez
|
r973 | self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock | ||
| r1263 | self.dataOut.profileIndex = self.profileIndex | |||
|
Jose Chavez
|
r973 | |||
| self.__bufferIndex += self.__nSamples | ||||
| r1263 | self.profileIndex += 1 | |||
|
Jose Chavez
|
r973 | |||
| if self.profileIndex == self.dataOut.nProfiles: | ||||
| self.profileIndex = 0 | ||||
| return True | ||||
| def printInfo(self): | ||||
| ''' | ||||
| ''' | ||||
| if self.__printInfo == False: | ||||
| return | ||||
|
Jose Chavez
|
r981 | # self.systemHeaderObj.printInfo() | ||
| # self.radarControllerHeaderObj.printInfo() | ||||
|
Jose Chavez
|
r973 | |||
| self.__printInfo = False | ||||
| def printNumberOfBlock(self): | ||||
| ''' | ||||
| ''' | ||||
| return | ||||
|
Jose Chavez
|
r981 | # print self.profileIndex | ||
|
Jose Chavez
|
r973 | |||
| def run(self, **kwargs): | ||||
| ''' | ||||
| This method will be called many times so here you should put all your code | ||||
| ''' | ||||
| r1263 | ||||
|
Jose Chavez
|
r973 | if not self.isConfig: | ||
| self.setup(**kwargs) | ||||
|
José Chávez
|
r998 | #self.i = self.i+1 | ||
|
Jose Chavez
|
r973 | self.getData(seconds=self.__delay) | ||
| r1263 | ||||
|
Jose Chavez
|
r973 | return | ||
|
Juan C. Espinoza
|
r1287 | @MPDecorator | ||
|
Jose Chavez
|
r973 | class DigitalRFWriter(Operation): | ||
| ''' | ||||
| classdocs | ||||
| ''' | ||||
| def __init__(self, **kwargs): | ||||
| ''' | ||||
| Constructor | ||||
| ''' | ||||
| Operation.__init__(self, **kwargs) | ||||
|
Jose Chavez
|
r981 | self.metadata_dict = {} | ||
| r1263 | self.dataOut = None | |||
| self.dtype = None | ||||
| self.oldAverage = 0 | ||||
|
Jose Chavez
|
r991 | |||
| def setHeader(self): | ||||
| r1263 | self.metadata_dict['frequency'] = self.dataOut.frequency | |||
| self.metadata_dict['timezone'] = self.dataOut.timeZone | ||||
| self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype) | ||||
| self.metadata_dict['nProfiles'] = self.dataOut.nProfiles | ||||
| self.metadata_dict['heightList'] = self.dataOut.heightList | ||||
| self.metadata_dict['channelList'] = self.dataOut.channelList | ||||
|
Jose Chavez
|
r991 | self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData | ||
| self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData | ||||
| r1263 | self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT | |||
| self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime | ||||
| self.metadata_dict['nCohInt'] = self.dataOut.nCohInt | ||||
| self.metadata_dict['type'] = self.dataOut.type | ||||
| self.metadata_dict['flagDataAsBlock']= getattr( | ||||
|
José Chávez
|
r1120 | self.dataOut, 'flagDataAsBlock', None) # chequear | ||
|
Jose Chavez
|
r982 | |||
|
Jose Chavez
|
r991 | def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'): | ||
|
Jose Chavez
|
r973 | ''' | ||
| In this method we should set all initial parameters. | ||||
| Input: | ||||
|
Jose Chavez
|
r981 | dataOut: Input data will also be outputa data | ||
|
Jose Chavez
|
r973 | ''' | ||
|
Jose Chavez
|
r991 | self.setHeader() | ||
| r1263 | self.__ippSeconds = dataOut.ippSeconds | |||
| self.__deltaH = dataOut.getDeltaH() | ||||
|
José Chávez
|
r1092 | self.__sample_rate = 1e6 * 0.15 / self.__deltaH | ||
| r1263 | self.__dtype = dataOut.dtype | |||
|
Jose Chavez
|
r980 | if len(dataOut.dtype) == 2: | ||
| self.__dtype = dataOut.dtype[0] | ||||
| r1263 | self.__nSamples = dataOut.systemHeaderObj.nSamples | |||
|
Jose Chavez
|
r980 | self.__nProfiles = dataOut.nProfiles | ||
|
José Chávez
|
r998 | |||
|
José Chávez
|
r1120 | if self.dataOut.type != 'Voltage': | ||
| raise 'Digital RF cannot be used with this data type' | ||||
| self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len( | ||||
| self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)]) | ||||
| else: | ||||
| self.arr_data = numpy.ones((self.__nSamples, len( | ||||
| self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)]) | ||||
|
Jose Chavez
|
r980 | |||
| r1263 | file_cadence_millisecs = 1000 | |||
|
Jose Chavez
|
r980 | |||
| r1263 | sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator() | |||
| sample_rate_numerator = int(sample_rate_fraction.numerator) | ||||
|
George Yong
|
r1167 | sample_rate_denominator = int(sample_rate_fraction.denominator) | ||
| r1263 | start_global_index = dataOut.utctime * self.__sample_rate | |||
|
José Chávez
|
r1092 | |||
| r1263 | uuid = 'prueba' | |||
|
José Chávez
|
r1103 | compression_level = 0 | ||
| r1263 | checksum = False | |||
| is_complex = True | ||||
| num_subchannels = len(dataOut.channelList) | ||||
| is_continuous = True | ||||
| marching_periods = False | ||||
|
Jose Chavez
|
r979 | |||
|
Jose Chavez
|
r991 | self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence, | ||
|
José Chávez
|
r1092 | fileCadence, start_global_index, | ||
| sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum, | ||||
| is_complex, num_subchannels, is_continuous, marching_periods) | ||||
| r1263 | metadata_dir = os.path.join(path, 'metadata') | |||
|
Jose Chavez
|
r980 | os.system('mkdir %s' % (metadata_dir)) | ||
|
José Chávez
|
r1092 | self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1, # 236, file_cadence_millisecs / 1000 | ||
| sample_rate_numerator, sample_rate_denominator, | ||||
| metadataFile) | ||||
| r1263 | self.isConfig = True | |||
|
Jose Chavez
|
r980 | self.currentSample = 0 | ||
| r1263 | self.oldAverage = 0 | |||
| self.count = 0 | ||||
|
Jose Chavez
|
r973 | return | ||
|
José Chávez
|
r1092 | |||
|
Jose Chavez
|
r981 | def writeMetadata(self): | ||
| r1263 | start_idx = self.__sample_rate * self.dataOut.utctime | |||
|
José Chávez
|
r1092 | |||
| r1263 | self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict( | |||
|
José Chávez
|
r1092 | ) | ||
| self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict( | ||||
| ) | ||||
| r1263 | self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict( | |||
|
José Chávez
|
r1092 | ) | ||
|
Jose Chavez
|
r981 | self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict) | ||
| return | ||||
|
Jose Chavez
|
r973 | |||
|
Jose Chavez
|
r991 | def timeit(self, toExecute): | ||
|
Jose Chavez
|
r985 | t0 = time() | ||
|
Jose Chavez
|
r991 | toExecute() | ||
| r1263 | self.executionTime = time() - t0 | |||
|
José Chávez
|
r1092 | if self.oldAverage is None: | ||
| self.oldAverage = self.executionTime | ||||
| r1263 | self.oldAverage = (self.executionTime + self.count * | |||
|
José Chávez
|
r1092 | self.oldAverage) / (self.count + 1.0) | ||
| r1263 | self.count = self.count + 1.0 | |||
|
Jose Chavez
|
r991 | return | ||
| def writeData(self): | ||||
|
José Chávez
|
r1120 | if self.dataOut.type != 'Voltage': | ||
| raise 'Digital RF cannot be used with this data type' | ||||
|
Jose Chavez
|
r991 | for channel in self.dataOut.channelList: | ||
|
José Chávez
|
r1120 | for i in range(self.dataOut.nFFTPoints): | ||
| self.arr_data[1][channel * self.dataOut.nFFTPoints + | ||||
| i]['r'] = self.dataOut.data[channel][i].real | ||||
| self.arr_data[1][channel * self.dataOut.nFFTPoints + | ||||
| i]['i'] = self.dataOut.data[channel][i].imag | ||||
| else: | ||||
| for i in range(self.dataOut.systemHeaderObj.nSamples): | ||||
| for channel in self.dataOut.channelList: | ||||
| self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real | ||||
| self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag | ||||
|
Jose Chavez
|
r991 | |||
| def f(): return self.digitalWriteObj.rf_write(self.arr_data) | ||||
| self.timeit(f) | ||||
|
José Chávez
|
r1092 | |||
|
Jose Chavez
|
r981 | return | ||
|
José Chávez
|
r1092 | |||
|
José Chávez
|
r1103 | def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs): | ||
|
Jose Chavez
|
r973 | ''' | ||
| This method will be called many times so here you should put all your code | ||||
| Inputs: | ||||
|
Jose Chavez
|
r981 | dataOut: object with the data | ||
|
Jose Chavez
|
r973 | ''' | ||
|
Jose Chavez
|
r981 | # print dataOut.__dict__ | ||
|
Jose Chavez
|
r980 | self.dataOut = dataOut | ||
|
Jose Chavez
|
r973 | if not self.isConfig: | ||
|
José Chávez
|
r1092 | self.setup(dataOut, path, frequency, fileCadence, | ||
| dirCadence, metadataCadence, **kwargs) | ||||
|
José Chávez
|
r998 | self.writeMetadata() | ||
|
Jose Chavez
|
r979 | |||
|
Jose Chavez
|
r981 | self.writeData() | ||
|
José Chávez
|
r1092 | |||
|
José Chávez
|
r998 | ## self.currentSample += 1 | ||
|
José Chávez
|
r1092 | # if self.dataOut.flagDataAsBlock or self.currentSample == 1: | ||
| # self.writeMetadata() | ||||
|
José Chávez
|
r998 | ## if self.currentSample == self.__nProfiles: self.currentSample = 0 | ||
|
Jose Chavez
|
r981 | |||
| r1263 | return dataOut# en la version 2.7 no aparece este return | |||
|
George Yong
|
r1192 | |||
|
Jose Chavez
|
r980 | def close(self): | ||
|
George Yong
|
r1167 | print('[Writing] - Closing files ') | ||
| print('Average of writing to digital rf format is ', self.oldAverage * 1000) | ||||
|
Jose Chavez
|
r981 | try: | ||
| self.digitalWriteObj.close() | ||||
| except: | ||||
| pass | ||||
