From ce24778ee4ae8eadb4bead64e669a4b4885fa12d 2020-07-13 03:42:54 From: Juan C. Espinoza Date: 2020-07-13 03:42:54 Subject: [PATCH] Fix ScpecraWriter and CohInt attribute --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a386d7..ab3a8ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ -## CHANGELOG: +# CHANGELOG: -### 3.0 +## 3.0 * Python 3.x & 2.X compatible * New architecture with multiprocessing support * Add @MPDecorator for multiprocessing Operations (Plots, Writers and Publishers) @@ -9,7 +9,7 @@ * Clean controller to optimize scripts (format & optype are no longer required) * New GUI with dinamic load of Units and operations (use Kivy framework) -### 2.3 +## 2.3 * Added support for Madrigal formats (reading/writing). * Added support for reading BLTR parameters (*.sswma). * Added support for reading Julia format (*.dat). @@ -30,83 +30,83 @@ * Updated README for MAC OS GUI installation. * Setup now installs numpy. -### 2.2.6 +## 2.2.6 * Graphics generated by the GUI are now the same as generated by scripts. Issue #1074. * Added support for C extensions. * Function `hildebrand_sehkon` optimized with a C wrapper. * Numpy version updated. * Migration to GIT. -### 2.2.5: +## 2.2.5: * splitProfiles and combineProfiles modules were added to VoltageProc and Signal Chain GUI. * nProfiles of USRP data (hdf5) is the number of profiles thera are in one second. * jroPlotter works directly with data objects instead of dictionaries * script "schain" was added to Signal Chain installer -### 2.2.4.1: +## 2.2.4.1: * jroIO_usrp.py is update to read Sandra's data * decimation in Spectra and RTI plots is always enabled. * time* window option added to GUI -### 2.2.4: +## 2.2.4: * jroproc_spectra_lags.py added to schainpy * Bug fixed in schainGUI: ProcUnit was created with the same id in some cases. * Bug fixed in jroHeaderIO: Header size validation. -### 2.2.3.1: +## 2.2.3.1: * Filtering block by time has been added. * Bug fixed plotting RTI, CoherenceMap and others using xmin and xmax parameters. The first day worked properly but the next days did not. -### 2.2.3: +## 2.2.3: * Bug fixed in GUI: Error getting(reading) Code value * Bug fixed in GUI: Flip option always needs channelList field * Bug fixed in jrodata: when one branch modified a value in "dataOut" (example: dataOut.code) this value was modified for every branch (because this was a reference). It was modified in data.copy() * Bug fixed in jroproc_voltage.profileSelector(): rangeList replaces to profileRangeList. -### 2.2.2: +## 2.2.2: * VoltageProc: ProfileSelector, Reshape, Decoder with nTxs!=1 and getblock=True was tested * Rawdata and testRawdata.py added to Signal Chain project -### 2.2.1: +## 2.2.1: * Bugs fixed in GUI * Views were improved in GUI * Support to MST* ISR experiments * Bug fixed getting noise using hyldebrant. (minimum number of points > 20%) * handleError added to jroplotter.py -### 2.2.0: +## 2.2.0: * GUI: use of external plotter * Compatible with matplotlib 1.5.0 -### 2.1.5: +## 2.1.5: * serializer module added to Signal Chain * jroplotter.py added to Signal Chain -### 2.1.4.2: +## 2.1.4.2: * A new Plotter Class was added * Project.start() does not accept filename as a parameter anymore -### 2.1.4.1: +## 2.1.4.1: * Send notifications when an error different to ValueError is detected -### 2.1.4: +## 2.1.4: * Sending error notifications to signal chain administrator * Login to email server added -### 2.1.3.3: +## 2.1.3.3: * Colored Button Icons were added to GUI -### 2.1.3.2: +## 2.1.3.2: * GUI: user interaction enhanced * controller_api.py: Safe access to ControllerThead -### 2.1.3.1: +## 2.1.3.1: * GUI: every icon were resized * jroproc_voltage.py: Print a message when "Read from code" option is selected and the code is not defined inside data file -### 2.1.3: +## 2.1.3: * jroplot_heispectra.py: SpectraHeisScope was not showing the right channels * jroproc_voltage.py: Bug fixed selecting profiles (self.nProfiles took a wrong value), Bug fixed selecting heights by block (selecting profiles instead heights) @@ -114,7 +114,7 @@ was modified for every branch (because this was a reference). It was modified in * jroIO_heispectra.py: Bug fixed in FitsReader. Using local Fits instance instead schainpy.mode.data.jrodata.Fits. * jroIO_heispectra.py: Channel index list does not exist. -### 2.1.2: +## 2.1.2: * jroutils_ftp.py: Bug fixed, Any error sending file stopped the Server Thread Server thread opens and closes remote server each time file list is sent * jroplot_spectra.py: Noise path was not being created when noise data is saved. diff --git a/schainpy/model/data/jrodata.py b/schainpy/model/data/jrodata.py index 41d8746..ba1c84a 100644 --- a/schainpy/model/data/jrodata.py +++ b/schainpy/model/data/jrodata.py @@ -388,6 +388,7 @@ class Voltage(JROData): self.errorCount = None self.nCohInt = None self.blocksize = None + self.flagCohInt = False self.flagDecodeData = False # asumo q la data no esta decodificada self.flagDeflipData = False # asumo q la data no esta sin flip self.flagShiftFFT = False diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index d9ad1f4..463d0b0 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -899,7 +899,8 @@ class JRODataReader(Reader): def readNextBlock(self): while True: - self.__setNewBlock() + if not(self.__setNewBlock()): + continue if not(self.readBlock()): return 0 @@ -954,55 +955,49 @@ class JRODataReader(Reader): # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels) self.getBlockDimension() - def verifyFile(self, filename, msgFlag=True): + def verifyFile(self, filename): - msg = None + flag = True try: fp = open(filename, 'rb') except IOError: - - if msgFlag: - print("[Reading] File %s can't be opened" % (filename)) - + log.error("File {} can't be opened".format(filename), self.name) return False - if self.waitDataBlock(0): - basicHeaderObj = BasicHeader(LOCALTIME) - systemHeaderObj = SystemHeader() - radarControllerHeaderObj = RadarControllerHeader() - processingHeaderObj = ProcessingHeader() - - if not(basicHeaderObj.read(fp)): - fp.close() - return False - - if not(systemHeaderObj.read(fp)): - fp.close() - return False - - if not(radarControllerHeaderObj.read(fp)): - fp.close() - return False - - if not(processingHeaderObj.read(fp)): - fp.close() - return False - - if not self.online: - dt1 = basicHeaderObj.datatime - fp.seek(self.fileSize-processingHeaderObj.blockSize-24) + if self.online and self.waitDataBlock(0): + pass + + basicHeaderObj = BasicHeader(LOCALTIME) + systemHeaderObj = SystemHeader() + radarControllerHeaderObj = RadarControllerHeader() + processingHeaderObj = ProcessingHeader() + + if not(basicHeaderObj.read(fp)): + flag = False + if not(systemHeaderObj.read(fp)): + flag = False + if not(radarControllerHeaderObj.read(fp)): + flag = False + if not(processingHeaderObj.read(fp)): + flag = False + if not self.online: + dt1 = basicHeaderObj.datatime + pos = self.fileSize-processingHeaderObj.blockSize-24 + if pos<0: + flag = False + log.error('Invalid size for file: {}'.format(self.filename), self.name) + else: + fp.seek(pos) if not(basicHeaderObj.read(fp)): - fp.close() - return False - dt2 = basicHeaderObj.datatime - if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \ - self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime): - return False + flag = False + dt2 = basicHeaderObj.datatime + if not self.isDateTimeInRange(dt1, self.startDate, self.endDate, self.startTime, self.endTime) and not \ + self.isDateTimeInRange(dt2, self.startDate, self.endDate, self.startTime, self.endTime): + flag = False fp.close() - - return True + return flag def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False): diff --git a/schainpy/model/io/jroIO_spectra.py b/schainpy/model/io/jroIO_spectra.py index 81d7972..589c9b6 100644 --- a/schainpy/model/io/jroIO_spectra.py +++ b/schainpy/model/io/jroIO_spectra.py @@ -358,7 +358,7 @@ class SpectraWriter(JRODataWriter, Operation): spc = numpy.transpose( self.data_spc, (0,2,1) ) if not self.processingHeaderObj.shif_fft: - spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones + spc = numpy.roll( spc, int(self.processingHeaderObj.profilesPerBlock/2), axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones data = spc.reshape((-1)) data = data.astype(self.dtype[0]) data.tofile(self.fp) @@ -369,7 +369,7 @@ class SpectraWriter(JRODataWriter, Operation): data = numpy.zeros( numpy.shape(cspc), self.dtype ) #print 'data.shape', self.shape_cspc_Buffer if not self.processingHeaderObj.shif_fft: - cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones + cspc = numpy.roll( cspc, int(self.processingHeaderObj.profilesPerBlock/2), axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones data['real'] = cspc.real data['imag'] = cspc.imag data = data.reshape((-1)) diff --git a/schainpy/model/proc/jroproc_base.py b/schainpy/model/proc/jroproc_base.py index c359084..62cb9d6 100644 --- a/schainpy/model/proc/jroproc_base.py +++ b/schainpy/model/proc/jroproc_base.py @@ -74,7 +74,7 @@ class ProcessingUnit(object): elif 'SchainError' in err: log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), self.name) else: - log.error(err.split('\n')[-2], self.name) + log.error(err, self.name) self.dataOut.error = True for op, optype, opkwargs in self.operations: @@ -193,7 +193,7 @@ def MPDecorator(BaseClass): BaseClass.run(self, dataOut, **self.kwargs) except: err = traceback.format_exc() - log.error(err.split('\n')[-2], self.name) + log.error(err, self.name) else: break diff --git a/schainpy/model/proc/jroproc_voltage.py b/schainpy/model/proc/jroproc_voltage.py index 13b8382..4f4748a 100644 --- a/schainpy/model/proc/jroproc_voltage.py +++ b/schainpy/model/proc/jroproc_voltage.py @@ -435,8 +435,6 @@ class CohInt(Operation): Operation.__init__(self, **kwargs) - # self.isConfig = False - def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False): """ Set the parameters of the integration class. @@ -670,7 +668,9 @@ class CohInt(Operation): if self.__dataReady: dataOut.data = avgdata - dataOut.nCohInt *= self.n + if not dataOut.flagCohInt: + dataOut.nCohInt *= self.n + dataOut.flagCohInt = True dataOut.utctime = avgdatatime # print avgdata, avgdatatime # raise