jroIO_digitalRF.py
799 lines
| 27.9 KiB
| text/x-python
|
PythonLexer
Jose Chavez
|
r979 | ||
|
Jose Chavez
|
r973 | ''' | |
| Created on Jul 3, 2014 | |||
| @author: roj-idl71 | |||
| ''' | |||
|
Jose Chavez
|
r991 | # SUBCHANNELS EN VEZ DE CHANNELS | |
| # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO | |||
| # ACTUALIZACION DE VERSION | |||
| # HEADERS | |||
| # MODULO DE ESCRITURA | |||
| # METADATA | |||
|
Jose Chavez
|
r973 | import os | |
| import datetime | |||
| import numpy | |||
|
Jose Chavez
|
r985 | import timeit | |
|
Jose Chavez
|
r979 | from fractions import Fraction | |
|
Jose Chavez
|
r973 | ||
| try: | |||
| from gevent import sleep | |||
| except: | |||
| from time import sleep | |||
| from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader | |||
| from schainpy.model.data.jrodata import Voltage | |||
| from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation | |||
|
Jose Chavez
|
r981 | from time import time | |
|
Jose Chavez
|
r985 | ||
George Yong
|
r1167 | import pickle | |
|
Jose Chavez
|
r973 | try: | |
| import digital_rf | |||
| except: | |||
|
George Yong
|
r1167 | print('You should install "digital_rf" module if you want to read Digital RF data') | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r973 | class DigitalRFReader(ProcessingUnit): | |
| ''' | |||
| classdocs | |||
| ''' | |||
| def __init__(self, **kwargs): | |||
| ''' | |||
| Constructor | |||
| ''' | |||
| ProcessingUnit.__init__(self, **kwargs) | |||
| self.dataOut = Voltage() | |||
| self.__printInfo = True | |||
| self.__flagDiscontinuousBlock = False | |||
| self.__bufferIndex = 9999999 | |||
| self.__ippKm = None | |||
| self.__codeType = 0 | |||
| self.__nCode = None | |||
| self.__nBaud = None | |||
| self.__code = None | |||
|
José Chávez
|
r1054 | self.dtype = None | |
|
José Chávez
|
r1103 | self.oldAverage = None | |
|
Jose Chavez
|
r973 | ||
|
Jose Chavez
|
r991 | def close(self): | |
|
George Yong
|
r1167 | print('Average of writing to digital rf format is ', self.oldAverage * 1000) | |
|
Jose Chavez
|
r991 | return | |
|
Jose Chavez
|
r973 | def __getCurrentSecond(self): | |
|
José Chávez
|
r1092 | return self.__thisUnixSample / self.__sample_rate | |
|
Jose Chavez
|
r973 | ||
| thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.") | |||
| def __setFileHeader(self): | |||
| ''' | |||
| In this method will be initialized every parameter of dataOut object (header, no data) | |||
| ''' | |||
|
José Chávez
|
r1092 | ippSeconds = 1.0 * self.__nSamples / self.__sample_rate | |
| nProfiles = 1.0 / ippSeconds # Number of profiles in one second | |||
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1054 | try: | |
|
José Chávez
|
r1092 | self.dataOut.radarControllerHeaderObj = RadarControllerHeader( | |
| self.__radarControllerHeader) | |||
|
José Chávez
|
r1054 | except: | |
| self.dataOut.radarControllerHeaderObj = RadarControllerHeader( | |||
|
José Chávez
|
r1092 | txA=0, | |
| txB=0, | |||
| nWindows=1, | |||
| nHeights=self.__nSamples, | |||
| firstHeight=self.__firstHeigth, | |||
| deltaHeight=self.__deltaHeigth, | |||
| codeType=self.__codeType, | |||
| nCode=self.__nCode, nBaud=self.__nBaud, | |||
| code=self.__code) | |||
|
José Chávez
|
r1054 | try: | |
| self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader) | |||
| except: | |||
| self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples, | |||
| nProfiles=nProfiles, | |||
|
José Chávez
|
r1092 | nChannels=len( | |
| self.__channelList), | |||
|
José Chávez
|
r1054 | adcResolution=14) | |
|
Jose Chavez
|
r973 | self.dataOut.type = "Voltage" | |
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r973 | self.dataOut.data = None | |
|
Jose Chavez
|
r981 | self.dataOut.dtype = self.dtype | |
|
Jose Chavez
|
r973 | ||
|
Jose Chavez
|
r981 | # self.dataOut.nChannels = 0 | |
|
Jose Chavez
|
r973 | ||
|
Jose Chavez
|
r981 | # self.dataOut.nHeights = 0 | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1069 | self.dataOut.nProfiles = int(nProfiles) | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | self.dataOut.heightList = self.__firstHeigth + \ | |
| numpy.arange(self.__nSamples, dtype=numpy.float) * \ | |||
| self.__deltaHeigth | |||
|
Jose Chavez
|
r973 | ||
|
George Yong
|
r1167 | self.dataOut.channelList = list(range(self.__num_subchannels)) | |
|
Jose Chavez
|
r973 | ||
| self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights() | |||
|
Jose Chavez
|
r981 | # self.dataOut.channelIndexList = None | |
|
Jose Chavez
|
r973 | ||
| self.dataOut.flagNoData = True | |||
|
Jose Chavez
|
r982 | ||
|
Jose Chavez
|
r981 | self.dataOut.flagDataAsBlock = False | |
| # Set to TRUE if the data is discontinuous | |||
|
Jose Chavez
|
r973 | self.dataOut.flagDiscontinuousBlock = False | |
| self.dataOut.utctime = None | |||
|
José Chávez
|
r1092 | # timezone like jroheader, difference in minutes between UTC and localtime | |
| self.dataOut.timeZone = self.__timezone / 60 | |||
|
Jose Chavez
|
r973 | ||
| self.dataOut.dstFlag = 0 | |||
| self.dataOut.errorCount = 0 | |||
|
José Chávez
|
r1054 | try: | |
|
José Chávez
|
r1092 | self.dataOut.nCohInt = self.fixed_metadata_dict.get( | |
| 'nCohInt', self.nCohInt) | |||
|
Jose Chavez
|
r982 | ||
|
José Chávez
|
r1092 | # asumo que la data esta decodificada | |
| self.dataOut.flagDecodeData = self.fixed_metadata_dict.get( | |||
| 'flagDecodeData', self.flagDecodeData) | |||
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | # asumo que la data esta sin flip | |
| self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData'] | |||
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1054 | self.dataOut.flagShiftFFT = self.fixed_metadata_dict['flagShiftFFT'] | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1054 | self.dataOut.useLocalTime = self.fixed_metadata_dict['useLocalTime'] | |
| except: | |||
| pass | |||
|
Jose Chavez
|
r973 | ||
| self.dataOut.ippSeconds = ippSeconds | |||
|
Jose Chavez
|
r981 | # Time interval between profiles | |
| # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt | |||
|
Jose Chavez
|
r973 | ||
| self.dataOut.frequency = self.__frequency | |||
| self.dataOut.realtime = self.__online | |||
| def findDatafiles(self, path, startDate=None, endDate=None): | |||
| if not os.path.isdir(path): | |||
| return [] | |||
| try: | |||
|
José Chávez
|
r1092 | digitalReadObj = digital_rf.DigitalRFReader( | |
| path, load_all_metadata=True) | |||
|
Jose Chavez
|
r973 | except: | |
| digitalReadObj = digital_rf.DigitalRFReader(path) | |||
| channelNameList = digitalReadObj.get_channels() | |||
| if not channelNameList: | |||
| return [] | |||
| metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0]) | |||
| sample_rate = metadata_dict['sample_rate'][0] | |||
| this_metadata_file = digitalReadObj.get_metadata(channelNameList[0]) | |||
| try: | |||
| timezone = this_metadata_file['timezone'].value | |||
| except: | |||
| timezone = 0 | |||
|
José Chávez
|
r1092 | startUTCSecond, endUTCSecond = digitalReadObj.get_bounds( | |
| channelNameList[0]) / sample_rate - timezone | |||
|
Jose Chavez
|
r973 | ||
| startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond) | |||
| endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond) | |||
| if not startDate: | |||
| startDate = startDatetime.date() | |||
| if not endDate: | |||
| endDate = endDatatime.date() | |||
| dateList = [] | |||
| thisDatetime = startDatetime | |||
|
José Chávez
|
r1092 | while(thisDatetime <= endDatatime): | |
|
Jose Chavez
|
r973 | ||
| thisDate = thisDatetime.date() | |||
| if thisDate < startDate: | |||
| continue | |||
| if thisDate > endDate: | |||
| break | |||
| dateList.append(thisDate) | |||
| thisDatetime += datetime.timedelta(1) | |||
| return dateList | |||
|
José Chávez
|
r1092 | def setup(self, path=None, | |
| startDate=None, | |||
| endDate=None, | |||
| startTime=datetime.time(0, 0, 0), | |||
| endTime=datetime.time(23, 59, 59), | |||
| channelList=None, | |||
| nSamples=None, | |||
| online=False, | |||
| delay=60, | |||
| buffer_size=1024, | |||
| ippKm=None, | |||
| nCohInt=1, | |||
| nCode=1, | |||
| nBaud=1, | |||
| flagDecodeData=False, | |||
| code=numpy.ones((1, 1), dtype=numpy.int), | |||
| **kwargs): | |||
|
Jose Chavez
|
r973 | ''' | |
| In this method we should set all initial parameters. | |||
| Inputs: | |||
| path | |||
| startDate | |||
| endDate | |||
| startTime | |||
| endTime | |||
| set | |||
| expLabel | |||
| ext | |||
| online | |||
| delay | |||
| ''' | |||
|
José Chávez
|
r1092 | self.nCohInt = nCohInt | |
| self.flagDecodeData = flagDecodeData | |||
|
José Chávez
|
r998 | self.i = 0 | |
|
Jose Chavez
|
r973 | if not os.path.isdir(path): | |
|
George Yong
|
r1167 | raise ValueError("[Reading] Directory %s does not exist" % path) | |
|
Jose Chavez
|
r973 | ||
| try: | |||
|
José Chávez
|
r1092 | self.digitalReadObj = digital_rf.DigitalRFReader( | |
| path, load_all_metadata=True) | |||
|
Jose Chavez
|
r973 | except: | |
| self.digitalReadObj = digital_rf.DigitalRFReader(path) | |||
| channelNameList = self.digitalReadObj.get_channels() | |||
| if not channelNameList: | |||
|
George Yong
|
r1167 | raise ValueError("[Reading] Directory %s does not have any files" % path) | |
|
Jose Chavez
|
r973 | ||
| if not channelList: | |||
|
George Yong
|
r1167 | channelList = list(range(len(channelNameList))) | |
|
Jose Chavez
|
r973 | ||
| ########## Reading metadata ###################### | |||
|
José Chávez
|
r1092 | top_properties = self.digitalReadObj.get_properties( | |
| channelNameList[channelList[0]]) | |||
|
Jose Chavez
|
r985 | ||
| self.__num_subchannels = top_properties['num_subchannels'] | |||
|
José Chávez
|
r1092 | self.__sample_rate = 1.0 * \ | |
| top_properties['sample_rate_numerator'] / \ | |||
| top_properties['sample_rate_denominator'] | |||
|
Jose Chavez
|
r981 | # self.__samples_per_file = top_properties['samples_per_file'][0] | |
|
José Chávez
|
r1092 | self.__deltaHeigth = 1e6 * 0.15 / self.__sample_rate # why 0.15? | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | this_metadata_file = self.digitalReadObj.get_digital_metadata( | |
| channelNameList[channelList[0]]) | |||
|
Jose Chavez
|
r973 | metadata_bounds = this_metadata_file.get_bounds() | |
|
José Chávez
|
r1092 | self.fixed_metadata_dict = this_metadata_file.read( | |
| metadata_bounds[0])[metadata_bounds[0]] # GET FIRST HEADER | |||
|
José Chávez
|
r1054 | ||
| try: | |||
| self.__processingHeader = self.fixed_metadata_dict['processingHeader'] | |||
| self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader'] | |||
| self.__systemHeader = self.fixed_metadata_dict['systemHeader'] | |||
|
George Yong
|
r1167 | self.dtype = pickle.loads(self.fixed_metadata_dict['dtype']) | |
|
José Chávez
|
r1054 | except: | |
| pass | |||
|
Jose Chavez
|
r973 | ||
| self.__frequency = None | |||
|
Jose Chavez
|
r981 | ||
|
José Chávez
|
r1054 | self.__frequency = self.fixed_metadata_dict.get('frequency', 1) | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1054 | self.__timezone = self.fixed_metadata_dict.get('timezone', 300) | |
|
Jose Chavez
|
r973 | ||
|
Jose Chavez
|
r981 | try: | |
|
Jose Chavez
|
r982 | nSamples = self.fixed_metadata_dict['nSamples'] | |
|
Jose Chavez
|
r981 | except: | |
| nSamples = None | |||
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r973 | self.__firstHeigth = 0 | |
| try: | |||
|
Jose Chavez
|
r981 | codeType = self.__radarControllerHeader['codeType'] | |
|
Jose Chavez
|
r973 | except: | |
| codeType = 0 | |||
|
José Chávez
|
r1054 | try: | |
| if codeType: | |||
| nCode = self.__radarControllerHeader['nCode'] | |||
| nBaud = self.__radarControllerHeader['nBaud'] | |||
| code = self.__radarControllerHeader['code'] | |||
| except: | |||
| pass | |||
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r973 | if not ippKm: | |
| try: | |||
|
Jose Chavez
|
r981 | # seconds to km | |
|
Jose Chavez
|
r982 | ippKm = self.__radarControllerHeader['ipp'] | |
|
Jose Chavez
|
r973 | except: | |
| ippKm = None | |||
| #################################################### | |||
|
José Chávez
|
r1054 | self.__ippKm = ippKm | |
|
Jose Chavez
|
r973 | startUTCSecond = None | |
| endUTCSecond = None | |||
| if startDate: | |||
| startDatetime = datetime.datetime.combine(startDate, startTime) | |||
|
José Chávez
|
r1092 | startUTCSecond = ( | |
| startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone | |||
|
Jose Chavez
|
r973 | ||
| if endDate: | |||
| endDatetime = datetime.datetime.combine(endDate, endTime) | |||
|
José Chávez
|
r1092 | endUTCSecond = (endDatetime - datetime.datetime(1970, | |
| 1, 1)).total_seconds() + self.__timezone | |||
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | start_index, end_index = self.digitalReadObj.get_bounds( | |
| channelNameList[channelList[0]]) | |||
|
Jose Chavez
|
r973 | ||
| if not startUTCSecond: | |||
|
José Chávez
|
r1092 | startUTCSecond = start_index / self.__sample_rate | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | if start_index > startUTCSecond * self.__sample_rate: | |
| startUTCSecond = start_index / self.__sample_rate | |||
|
Jose Chavez
|
r973 | ||
| if not endUTCSecond: | |||
|
José Chávez
|
r1092 | endUTCSecond = end_index / self.__sample_rate | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | if end_index < endUTCSecond * self.__sample_rate: | |
| endUTCSecond = end_index / self.__sample_rate | |||
|
Jose Chavez
|
r973 | if not nSamples: | |
| if not ippKm: | |||
|
George Yong
|
r1167 | raise ValueError("[Reading] nSamples or ippKm should be defined") | |
|
José Chávez
|
r1092 | nSamples = int(ippKm / (1e6 * 0.15 / self.__sample_rate)) | |
|
Jose Chavez
|
r973 | channelBoundList = [] | |
| channelNameListFiltered = [] | |||
| for thisIndexChannel in channelList: | |||
|
José Chávez
|
r1092 | thisChannelName = channelNameList[thisIndexChannel] | |
| start_index, end_index = self.digitalReadObj.get_bounds( | |||
| thisChannelName) | |||
|
Jose Chavez
|
r973 | channelBoundList.append((start_index, end_index)) | |
| channelNameListFiltered.append(thisChannelName) | |||
| self.profileIndex = 0 | |||
|
José Chávez
|
r1092 | self.i = 0 | |
|
Jose Chavez
|
r973 | self.__delay = delay | |
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r973 | self.__codeType = codeType | |
| self.__nCode = nCode | |||
| self.__nBaud = nBaud | |||
| self.__code = code | |||
| self.__datapath = path | |||
| self.__online = online | |||
| self.__channelList = channelList | |||
| self.__channelNameList = channelNameListFiltered | |||
| self.__channelBoundList = channelBoundList | |||
| self.__nSamples = nSamples | |||
|
George Yong
|
r1167 | self.__samples_to_read = int(nSamples) # FIJO: AHORA 40 | |
|
Jose Chavez
|
r973 | self.__nChannels = len(self.__channelList) | |
| self.__startUTCSecond = startUTCSecond | |||
| self.__endUTCSecond = endUTCSecond | |||
|
José Chávez
|
r1092 | self.__timeInterval = 1.0 * self.__samples_to_read / \ | |
| self.__sample_rate # Time interval | |||
|
Jose Chavez
|
r973 | ||
| if online: | |||
|
José Chávez
|
r1092 | # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read) | |
|
Jose Chavez
|
r973 | startUTCSecond = numpy.floor(endUTCSecond) | |
|
José Chávez
|
r1092 | # por que en el otro metodo lo primero q se hace es sumar samplestoread | |
|
George Yong
|
r1167 | self.__thisUnixSample = int( | |
|
José Chávez
|
r1092 | startUTCSecond * self.__sample_rate) - self.__samples_to_read | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | self.__data_buffer = numpy.zeros( | |
| (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex) | |||
|
Jose Chavez
|
r973 | ||
| self.__setFileHeader() | |||
| self.isConfig = True | |||
|
George Yong
|
r1167 | print("[Reading] Digital RF Data was found from %s to %s " % ( | |
|
José Chávez
|
r1092 | datetime.datetime.utcfromtimestamp( | |
| self.__startUTCSecond - self.__timezone), | |||
| datetime.datetime.utcfromtimestamp( | |||
| self.__endUTCSecond - self.__timezone) | |||
|
George Yong
|
r1167 | )) | |
|
José Chávez
|
r1092 | ||
|
George Yong
|
r1167 | print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone), | |
|
José Chávez
|
r1092 | datetime.datetime.utcfromtimestamp( | |
| endUTCSecond - self.__timezone) | |||
|
George Yong
|
r1167 | )) | |
|
Jose Chavez
|
r991 | self.oldAverage = None | |
| self.count = 0 | |||
| self.executionTime = 0 | |||
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r973 | def __reload(self): | |
| # print "%s not in range [%s, %s]" %( | |||
| # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), | |||
| # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone), | |||
| # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone) | |||
| # ) | |||
|
George Yong
|
r1167 | print("[Reading] reloading metadata ...") | |
|
Jose Chavez
|
r973 | ||
| try: | |||
| self.digitalReadObj.reload(complete_update=True) | |||
| except: | |||
| self.digitalReadObj.reload() | |||
|
José Chávez
|
r1092 | start_index, end_index = self.digitalReadObj.get_bounds( | |
| self.__channelNameList[self.__channelList[0]]) | |||
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | if start_index > self.__startUTCSecond * self.__sample_rate: | |
| self.__startUTCSecond = 1.0 * start_index / self.__sample_rate | |||
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | if end_index > self.__endUTCSecond * self.__sample_rate: | |
| self.__endUTCSecond = 1.0 * end_index / self.__sample_rate | |||
|
George Yong
|
r1167 | print() | |
| print("[Reading] New timerange found [%s, %s] " % ( | |||
|
José Chávez
|
r1092 | datetime.datetime.utcfromtimestamp( | |
| self.__startUTCSecond - self.__timezone), | |||
| datetime.datetime.utcfromtimestamp( | |||
| self.__endUTCSecond - self.__timezone) | |||
|
George Yong
|
r1167 | )) | |
|
Jose Chavez
|
r973 | ||
| return True | |||
| return False | |||
|
Jose Chavez
|
r991 | def timeit(self, toExecute): | |
| t0 = time() | |||
| toExecute() | |||
| self.executionTime = time() - t0 | |||
|
José Chávez
|
r1092 | if self.oldAverage is None: | |
| self.oldAverage = self.executionTime | |||
| self.oldAverage = (self.executionTime + self.count * | |||
| self.oldAverage) / (self.count + 1.0) | |||
|
Jose Chavez
|
r991 | self.count = self.count + 1.0 | |
| return | |||
|
José Chávez
|
r1092 | def __readNextBlock(self, seconds=30, volt_scale=1): | |
|
Jose Chavez
|
r973 | ''' | |
| ''' | |||
|
Jose Chavez
|
r981 | # Set the next data | |
|
Jose Chavez
|
r973 | self.__flagDiscontinuousBlock = False | |
| self.__thisUnixSample += self.__samples_to_read | |||
|
José Chávez
|
r1092 | if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate: | |
|
George Yong
|
r1167 | print("[Reading] There are no more data into selected time-range") | |
|
Jose Chavez
|
r973 | if self.__online: | |
| self.__reload() | |||
| else: | |||
| return False | |||
|
José Chávez
|
r1092 | if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate: | |
|
Jose Chavez
|
r973 | return False | |
|
José Chávez
|
r1092 | self.__thisUnixSample -= self.__samples_to_read | |
|
Jose Chavez
|
r973 | ||
| indexChannel = 0 | |||
| dataOk = False | |||
|
José Chávez
|
r1092 | for thisChannelName in self.__channelNameList: # TODO VARIOS CHANNELS? | |
|
Jose Chavez
|
r985 | for indexSubchannel in range(self.__num_subchannels): | |
| try: | |||
|
Jose Chavez
|
r991 | t0 = time() | |
|
Jose Chavez
|
r985 | result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample, | |
|
Jose Chavez
|
r991 | self.__samples_to_read, | |
| thisChannelName, sub_channel=indexSubchannel) | |||
| self.executionTime = time() - t0 | |||
|
José Chávez
|
r1092 | if self.oldAverage is None: | |
| self.oldAverage = self.executionTime | |||
| self.oldAverage = ( | |||
| self.executionTime + self.count * self.oldAverage) / (self.count + 1.0) | |||
|
Jose Chavez
|
r991 | self.count = self.count + 1.0 | |
|
José Chávez
|
r1092 | ||
|
George Yong
|
r1167 | except IOError as e: | |
|
José Chávez
|
r1092 | # read next profile | |
|
Jose Chavez
|
r985 | self.__flagDiscontinuousBlock = True | |
|
George Yong
|
r1167 | print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e) | |
|
Jose Chavez
|
r985 | break | |
|
Jose Chavez
|
r973 | ||
|
Jose Chavez
|
r985 | if result.shape[0] != self.__samples_to_read: | |
| self.__flagDiscontinuousBlock = True | |||
|
George Yong
|
r1167 | print("[Reading] %s: Too few samples were found, just %d/%d samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), | |
|
José Chávez
|
r1092 | result.shape[0], | |
|
George Yong
|
r1167 | self.__samples_to_read)) | |
|
Jose Chavez
|
r985 | break | |
|
Jose Chavez
|
r973 | ||
|
José Chávez
|
r1092 | self.__data_buffer[indexSubchannel, :] = result * volt_scale | |
|
Jose Chavez
|
r973 | ||
|
Jose Chavez
|
r985 | indexChannel += 1 | |
|
Jose Chavez
|
r973 | ||
|
Jose Chavez
|
r985 | dataOk = True | |
|
José Chávez
|
r1092 | ||
| self.__utctime = self.__thisUnixSample / self.__sample_rate | |||
|
Jose Chavez
|
r973 | ||
| if not dataOk: | |||
| return False | |||
|
George Yong
|
r1167 | print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), | |
|
José Chávez
|
r1092 | self.__samples_to_read, | |
|
George Yong
|
r1167 | self.__timeInterval)) | |
|
Jose Chavez
|
r973 | ||
| self.__bufferIndex = 0 | |||
| return True | |||
| def __isBufferEmpty(self): | |||
|
José Chávez
|
r1092 | return self.__bufferIndex > self.__samples_to_read - self.__nSamples # 40960 - 40 | |
|
Jose Chavez
|
r973 | ||
| def getData(self, seconds=30, nTries=5): | |||
| ''' | |||
| This method gets the data from files and put the data into the dataOut object | |||
| In addition, increase el the buffer counter in one. | |||
| Return: | |||
| data : retorna un perfil de voltages (alturas * canales) copiados desde el | |||
| buffer. Si no hay mas archivos a leer retorna None. | |||
| Affected: | |||
| self.dataOut | |||
| self.profileIndex | |||
| self.flagDiscontinuousBlock | |||
| self.flagIsNewBlock | |||
| ''' | |||
| err_counter = 0 | |||
| self.dataOut.flagNoData = True | |||
| if self.__isBufferEmpty(): | |||
| self.__flagDiscontinuousBlock = False | |||
| while True: | |||
| if self.__readNextBlock(): | |||
| break | |||
|
José Chávez
|
r1092 | if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate: | |
|
Jose Chavez
|
r973 | return False | |
| if self.__flagDiscontinuousBlock: | |||
|
George Yong
|
r1167 | print('[Reading] discontinuous block found ... continue with the next block') | |
|
Jose Chavez
|
r973 | continue | |
| if not self.__online: | |||
| return False | |||
| err_counter += 1 | |||
| if err_counter > nTries: | |||
| return False | |||
|
George Yong
|
r1167 | print('[Reading] waiting %d seconds to read a new block' % seconds) | |
|
Jose Chavez
|
r973 | sleep(seconds) | |
|
José Chávez
|
r1092 | self.dataOut.data = self.__data_buffer[:, | |
| self.__bufferIndex:self.__bufferIndex + self.__nSamples] | |||
| self.dataOut.utctime = ( | |||
| self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate | |||
|
Jose Chavez
|
r973 | self.dataOut.flagNoData = False | |
| self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock | |||
| self.dataOut.profileIndex = self.profileIndex | |||
| self.__bufferIndex += self.__nSamples | |||
| self.profileIndex += 1 | |||
| if self.profileIndex == self.dataOut.nProfiles: | |||
| self.profileIndex = 0 | |||
| return True | |||
| def printInfo(self): | |||
| ''' | |||
| ''' | |||
| if self.__printInfo == False: | |||
| return | |||
|
Jose Chavez
|
r981 | # self.systemHeaderObj.printInfo() | |
| # self.radarControllerHeaderObj.printInfo() | |||
|
Jose Chavez
|
r973 | ||
| self.__printInfo = False | |||
| def printNumberOfBlock(self): | |||
| ''' | |||
| ''' | |||
| return | |||
|
Jose Chavez
|
r981 | # print self.profileIndex | |
|
Jose Chavez
|
r973 | ||
| def run(self, **kwargs): | |||
| ''' | |||
| This method will be called many times so here you should put all your code | |||
| ''' | |||
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r973 | if not self.isConfig: | |
| self.setup(**kwargs) | |||
|
José Chávez
|
r998 | #self.i = self.i+1 | |
|
Jose Chavez
|
r973 | self.getData(seconds=self.__delay) | |
| return | |||
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r973 | class DigitalRFWriter(Operation): | |
| ''' | |||
| classdocs | |||
| ''' | |||
| def __init__(self, **kwargs): | |||
| ''' | |||
| Constructor | |||
| ''' | |||
| Operation.__init__(self, **kwargs) | |||
|
Jose Chavez
|
r981 | self.metadata_dict = {} | |
|
José Chávez
|
r1092 | self.dataOut = None | |
|
José Chávez
|
r1054 | self.dtype = None | |
|
José Chávez
|
r1103 | self.oldAverage = 0 | |
|
Jose Chavez
|
r991 | ||
| def setHeader(self): | |||
| self.metadata_dict['frequency'] = self.dataOut.frequency | |||
| self.metadata_dict['timezone'] = self.dataOut.timeZone | |||
|
George Yong
|
r1167 | self.metadata_dict['dtype'] = pickle.dumps(self.dataOut.dtype) | |
|
Jose Chavez
|
r991 | self.metadata_dict['nProfiles'] = self.dataOut.nProfiles | |
| self.metadata_dict['heightList'] = self.dataOut.heightList | |||
| self.metadata_dict['channelList'] = self.dataOut.channelList | |||
| self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData | |||
| self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData | |||
| self.metadata_dict['flagShiftFFT'] = self.dataOut.flagShiftFFT | |||
| self.metadata_dict['useLocalTime'] = self.dataOut.useLocalTime | |||
| self.metadata_dict['nCohInt'] = self.dataOut.nCohInt | |||
|
José Chávez
|
r1120 | self.metadata_dict['type'] = self.dataOut.type | |
| self.metadata_dict['flagDataAsBlock'] = getattr( | |||
| self.dataOut, 'flagDataAsBlock', None) # chequear | |||
|
Jose Chavez
|
r982 | ||
|
Jose Chavez
|
r991 | def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'): | |
|
Jose Chavez
|
r973 | ''' | |
| In this method we should set all initial parameters. | |||
| Input: | |||
|
Jose Chavez
|
r981 | dataOut: Input data will also be outputa data | |
|
Jose Chavez
|
r973 | ''' | |
|
Jose Chavez
|
r991 | self.setHeader() | |
|
Jose Chavez
|
r980 | self.__ippSeconds = dataOut.ippSeconds | |
| self.__deltaH = dataOut.getDeltaH() | |||
|
José Chávez
|
r1092 | self.__sample_rate = 1e6 * 0.15 / self.__deltaH | |
|
Jose Chavez
|
r980 | self.__dtype = dataOut.dtype | |
| if len(dataOut.dtype) == 2: | |||
| self.__dtype = dataOut.dtype[0] | |||
| self.__nSamples = dataOut.systemHeaderObj.nSamples | |||
| self.__nProfiles = dataOut.nProfiles | |||
|
José Chávez
|
r998 | ||
|
José Chávez
|
r1120 | if self.dataOut.type != 'Voltage': | |
| raise 'Digital RF cannot be used with this data type' | |||
| self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len( | |||
| self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)]) | |||
| else: | |||
| self.arr_data = numpy.ones((self.__nSamples, len( | |||
| self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)]) | |||
|
Jose Chavez
|
r980 | ||
|
José Chávez
|
r1103 | file_cadence_millisecs = 1000 | |
|
Jose Chavez
|
r980 | ||
|
Jose Chavez
|
r979 | sample_rate_fraction = Fraction(self.__sample_rate).limit_denominator() | |
|
George Yong
|
r1167 | sample_rate_numerator = int(sample_rate_fraction.numerator) | |
| sample_rate_denominator = int(sample_rate_fraction.denominator) | |||
|
Jose Chavez
|
r980 | start_global_index = dataOut.utctime * self.__sample_rate | |
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r979 | uuid = 'prueba' | |
|
José Chávez
|
r1103 | compression_level = 0 | |
|
Jose Chavez
|
r979 | checksum = False | |
| is_complex = True | |||
|
Jose Chavez
|
r985 | num_subchannels = len(dataOut.channelList) | |
|
Jose Chavez
|
r979 | is_continuous = True | |
| marching_periods = False | |||
|
Jose Chavez
|
r991 | self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence, | |
|
José Chávez
|
r1092 | fileCadence, start_global_index, | |
| sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum, | |||
| is_complex, num_subchannels, is_continuous, marching_periods) | |||
|
Jose Chavez
|
r980 | metadata_dir = os.path.join(path, 'metadata') | |
| os.system('mkdir %s' % (metadata_dir)) | |||
|
José Chávez
|
r1092 | self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1, # 236, file_cadence_millisecs / 1000 | |
| sample_rate_numerator, sample_rate_denominator, | |||
| metadataFile) | |||
|
Jose Chavez
|
r973 | self.isConfig = True | |
|
Jose Chavez
|
r980 | self.currentSample = 0 | |
|
Jose Chavez
|
r985 | self.oldAverage = 0 | |
| self.count = 0 | |||
|
Jose Chavez
|
r973 | return | |
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r981 | def writeMetadata(self): | |
| start_idx = self.__sample_rate * self.dataOut.utctime | |||
|
José Chávez
|
r1092 | ||
| self.metadata_dict['processingHeader'] = self.dataOut.processingHeaderObj.getAsDict( | |||
| ) | |||
| self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict( | |||
| ) | |||
| self.metadata_dict['systemHeader'] = self.dataOut.systemHeaderObj.getAsDict( | |||
| ) | |||
|
Jose Chavez
|
r981 | self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict) | |
| return | |||
|
Jose Chavez
|
r973 | ||
|
Jose Chavez
|
r991 | def timeit(self, toExecute): | |
|
Jose Chavez
|
r985 | t0 = time() | |
|
Jose Chavez
|
r991 | toExecute() | |
|
Jose Chavez
|
r985 | self.executionTime = time() - t0 | |
|
José Chávez
|
r1092 | if self.oldAverage is None: | |
| self.oldAverage = self.executionTime | |||
| self.oldAverage = (self.executionTime + self.count * | |||
| self.oldAverage) / (self.count + 1.0) | |||
|
Jose Chavez
|
r985 | self.count = self.count + 1.0 | |
|
Jose Chavez
|
r991 | return | |
| def writeData(self): | |||
|
José Chávez
|
r1120 | if self.dataOut.type != 'Voltage': | |
| raise 'Digital RF cannot be used with this data type' | |||
|
Jose Chavez
|
r991 | for channel in self.dataOut.channelList: | |
|
José Chávez
|
r1120 | for i in range(self.dataOut.nFFTPoints): | |
| self.arr_data[1][channel * self.dataOut.nFFTPoints + | |||
| i]['r'] = self.dataOut.data[channel][i].real | |||
| self.arr_data[1][channel * self.dataOut.nFFTPoints + | |||
| i]['i'] = self.dataOut.data[channel][i].imag | |||
| else: | |||
| for i in range(self.dataOut.systemHeaderObj.nSamples): | |||
| for channel in self.dataOut.channelList: | |||
| self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real | |||
| self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag | |||
|
Jose Chavez
|
r991 | ||
| def f(): return self.digitalWriteObj.rf_write(self.arr_data) | |||
| self.timeit(f) | |||
|
José Chávez
|
r1092 | ||
|
Jose Chavez
|
r981 | return | |
|
José Chávez
|
r1092 | ||
|
José Chávez
|
r1103 | def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs): | |
|
Jose Chavez
|
r973 | ''' | |
| This method will be called many times so here you should put all your code | |||
| Inputs: | |||
|
Jose Chavez
|
r981 | dataOut: object with the data | |
|
Jose Chavez
|
r973 | ''' | |
|
Jose Chavez
|
r981 | # print dataOut.__dict__ | |
|
Jose Chavez
|
r980 | self.dataOut = dataOut | |
|
Jose Chavez
|
r973 | if not self.isConfig: | |
|
José Chávez
|
r1092 | self.setup(dataOut, path, frequency, fileCadence, | |
| dirCadence, metadataCadence, **kwargs) | |||
|
José Chávez
|
r998 | self.writeMetadata() | |
|
Jose Chavez
|
r979 | ||
|
Jose Chavez
|
r981 | self.writeData() | |
|
José Chávez
|
r1092 | ||
|
José Chávez
|
r998 | ## self.currentSample += 1 | |
|
José Chávez
|
r1092 | # if self.dataOut.flagDataAsBlock or self.currentSample == 1: | |
| # self.writeMetadata() | |||
|
José Chávez
|
r998 | ## if self.currentSample == self.__nProfiles: self.currentSample = 0 | |
|
Jose Chavez
|
r981 | ||
|
Jose Chavez
|
r980 | def close(self): | |
|
George Yong
|
r1167 | print('[Writing] - Closing files ') | |
| print('Average of writing to digital rf format is ', self.oldAverage * 1000) | |||
|
Jose Chavez
|
r981 | try: | |
| self.digitalWriteObj.close() | |||
| except: | |||
| pass | |||
|
José Chávez
|
r1092 | ||
|
José Chávez
|
r1103 | ||
|
Jose Chavez
|
r981 | # raise | |
|
Jose Chavez
|
r973 | if __name__ == '__main__': | |
| readObj = DigitalRFReader() | |||
| while True: | |||
| readObj.run(path='/home/jchavez/jicamarca/mocked_data/') | |||
|
Jose Chavez
|
r981 | # readObj.printInfo() | |
|
George Yong
|
r1167 | # readObj.printNumberOfBlock() |