schain Commit - r1279:c53fe2a4a291 · Jicamarca Repository

Modificación a kmamisr para ejecutarse en la versión 3, creación de scripts con terminación v3 para difereciarlos, se comentó la linea #720 de JroIO_param.py debido a que reiniciaba la lista de archivos, ocasionando la reescritura del archivo hdf5. Alguna otra modificación aparente es producto de algunas variaciones en espacios al usar la función print()

joabAM -

r1279:c53fe2a4a291

parent child

Context file:

r1279:c53fe2a4a291

Collapse all files

The requested changes are too big and content was truncated. Show full diff

schainpy/controller.py +41 -36

                     nFiles = len(files)
                     if nFiles == 0:
                         continue
                     skip = int(math.ceil(nFiles / n))
                     while nFiles > cursor * skip:
                         rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
                                      skip=skip)
                     time.sleep(3)
             def wait(context):
                 time.sleep(1)
                 c = zmq.Context()
                 receiver = c.socket(zmq.SUB)
                 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
                 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
                 msg = receiver.recv_multipart()[1]
                 context.terminate()
                         parmElement.set('name', self.name)
                         parmElement.set('value', self.value)
                         parmElement.set('format', self.format)
                 def readXml(self, parmElement):
                     self.id = parmElement.get('id')
                     self.name = opElement.get('name')
                     self.type = opElement.get('type')
                     self.priority = opElement.get('priority')
                     self.project_id = str(project_id)
                     # Compatible with old signal chain version
                     # Use of 'run' method instead 'init'
                     self.id = None
                     self.datatype = None
                     self.name = None
                     self.inputId = None
                     self.opConfObjList = []
                     self.procUnitObj = None
                     self.opObjDict = {}
                     return self.id
                 def updateId(self, new_id):
                     '''
                     new_id = int(parentId) * 10 + (int(self.id) % 10)
                     new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
                         id sera el topico a publicar
                         inputId sera el topico a subscribirse
                     '''
                     # Compatible with old signal chain version
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     self.lock = lock
                     self.opConfObjList = []
                     self.addOperation(name='run', optype='self')
                 def removeOperations(self):
                     '''
                     className = eval(self.name)
+                    #print(self.name)
                     kwargs = self.getKwargs()
+                    #print(kwargs)
+                    #print("mark_a")
                     procUnitObj = className(self.id, self.inputId, self.project_id, self.err_queue, self.lock, 'ProcUnit', **kwargs)
+                    #print("mark_b")
                     log.success('creating process...', self.name)
                     for opConfObj in self.opConfObjList:
                         if opConfObj.type == 'self' and opConfObj.name == 'run':
                             continue
                         elif opConfObj.type == 'self':
                             opObj = getattr(procUnitObj, opConfObj.name)
                         else:
                             opObj = opConfObj.createObject()
                         log.success('adding operation: {}, type:{}'.format(
                             opConfObj.name,
                             opConfObj.type), self.name)
                         procUnitObj.addOperation(opConfObj, opObj)
                     procUnitObj.start()
                     self.procUnitObj = procUnitObj
                 def close(self):
                     for opConfObj in self.opConfObjList:
                 def getElementName(self):
                     return self.ELEMENTNAME
                 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
                           startTime='', endTime='', server=None, **kwargs):
                     kwargs deben ser trasmitidos en la instanciacion
                     '''
                     # Compatible with old signal chain version
                     if datatype == None and name == None:
                         raise ValueError('datatype or name should be defined')
                     self.datatype = datatype
                     if path != '':
                         self.path = os.path.abspath(path)
+                    print (self.path)
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.server = server
                     self.err_queue = err_queue
                     self.addRunOperation(**kwargs)
                 def update(self, **kwargs):
                 def addRunOperation(self, **kwargs):
                     opObj = self.addOperation(name='run', optype='self')
                     if self.server is None:
                         opObj.addParameter(
                     print('*' * 19)
                     print(' ')
                     self.id = str(id)
                     self.description = description
                     self.email = email
                     self.alarm = alarm
                     if name:
                     readUnitConfObj = ReadUnitConf()
                     readUnitConfObj.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
                     self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                     return readUnitConfObj
                 def addProcUnit(self, inputId='0', datatype=None, name=None):
                     idProcUnit = self.__getNewId()
                     procUnitConfObj = ProcUnitConf()
                     input_proc = self.procUnitConfObjDict[inputId]
                     procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue, input_proc.lock)
                     self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                     t = Thread(target=self.__monitor, args=(self.err_queue, self.ctx))
                     t.start()
                 def __monitor(self, queue, ctx):
                     import socket
                     procs = 0
                     err_msg = ''
                     while True:
                         msg = queue.get()
                         if '#_start_#' in msg:
                             procs -=1
                         else:
                             err_msg = msg
                         if procs == 0 or 'Traceback' in err_msg:
                             break
                         time.sleep(0.1)
                     if '|' in err_msg:
                         name, err = err_msg.split('|')
                         if 'SchainWarning' in err:
                             log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
                         else:
                             log.error(err, name)
                     else:
                         name, err = self.name, err_msg
                     time.sleep(2)
                     for conf in self.procUnitConfObjDict.values():
                             if confop.type == 'external':
                                 confop.opObj.terminate()
                         conf.procUnitObj.terminate()
                     ctx.term()
                     message = ''.join(err)
                             subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
                         a = Alarm(
                             modes=self.alarm,
                             email=self.email,
                             message=message,
                             subject=subject,
                     if not os.path.exists('/tmp/schain'):
                         os.mkdir('/tmp/schain')
                     self.ctx = zmq.Context()
                     xpub = self.ctx.socket(zmq.XPUB)
                     xpub.bind('ipc:///tmp/schain/{}_pub'.format(self.id))
                 def run(self):
                     log.success('Starting {}: {}'.format(self.name, self.id), tag='')
                     self.start_time = time.time()
                     self.createObjects()
                     self.setProxy()
                     log.success('{} Done (Time: {}s)'.format(
                         self.name,
                         time.time()-self.start_time), '')

schainpy/model/data/jrodata.py +13 -13

                 flagNoData = True
                 def copy(self, inputObj=None):
                     if inputObj == None:
                         return copy.deepcopy(self)
                     deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
                     velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
                     if self.nmodes:
                         return velrange/self.nmodes
                     else:
                 MAXNUMY = 100
                 def __init__(self, code, throttle_value, exp_code, buffering=True, snr=False):
                     self.key = code
                     self.throttle = throttle_value
                     self.exp_code = exp_code
                     return len(self.__times)
                 def __getitem__(self, key):
                     if key not in self.data:
                         raise KeyError(log.error('Missing key: {}'.format(key)))
                     if 'spc' in key or not self.buffering:
                         elif 'spc_moments' == plot:
                             plot = 'moments'
                         self.data[plot] = {}
                     if 'spc' in self.data or 'rti' in self.data or 'cspc' in self.data or 'moments' in self.data:
                         self.data['noise'] = {}
                         self.data['rti'] = {}
                             self.plottypes.append('noise')
                         if 'rti' not in self.plottypes:
                             self.plottypes.append('rti')
                 def shape(self, key):
                     '''
                     Get the shape of the one-element data for the given key
                     '''
                     Update data object with new dataOut
                     '''
                     if tm in self.__times:
                         return
                     self.profileIndex = dataOut.profileIndex
                     self.tm = tm
                     self.type = dataOut.type
                     self.parameters = getattr(dataOut, 'parameters', [])
                     if hasattr(dataOut, 'meta'):
                         self.meta.update(dataOut.meta)
                     self.pairs = dataOut.pairsList
                     self.interval = dataOut.getTimeInterval()
                     self.localtime = dataOut.useLocalTime
                     self.__heights.append(dataOut.heightList)
                     self.__all_heights.update(dataOut.heightList)
                     self.__times.append(tm)
                     for plot in self.plottypes:
                         if plot in ('spc', 'spc_moments'):
                             z = dataOut.data_spc/dataOut.normFactor
                         if plot == 'scope':
                             buffer = dataOut.data
                             self.flagDataAsBlock = dataOut.flagDataAsBlock
                             self.nProfiles = dataOut.nProfiles
                         if plot == 'spc':
                             self.data['spc'] = buffer
                         elif plot == 'cspc':
                     else:
                         meta['xrange'] = []
                     meta.update(self.meta)
                     ret['metadata'] = meta
                     return json.dumps(ret)

schainpy/model/data/jroheaderIO.py +2 -2

                 structure = SYSTEM_STRUCTURE
                 def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
                     self.size = 24
                     self.nSamples = nSamples
                     self.nProfiles = nProfiles
             def get_dtype_width(index):
-                return DTYPE_WIDTH[index]
  No newline at end of file
+                return DTYPE_WIDTH[index]

schainpy/model/graphics/jroplot_base.py +23 -25

                     self.__throttle_plot = apply_throttle(self.throttle)
                     self.data = PlotterData(
                         self.CODE, self.throttle, self.exp_code, self.buffering, snr=self.showSNR)
                     if self.plot_server:
                         if not self.plot_server.startswith('tcp://'):
                             self.plot_server = 'tcp://{}'.format(self.plot_server)
                     self.setup()
                     self.time_label = 'LT' if self.localtime else 'UTC'
                     if self.width is None:
                         self.width = 8
                             cmap = plt.get_cmap(self.colormap)
                         cmap.set_bad(self.bgcolor, 1.)
                         self.cmaps.append(cmap)
                     for fig in self.figures:
                         fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
                         fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
                                 xmax += time.timezone
                         else:
                             xmax = self.xmax
                     ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                     ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
                     #Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000])
                     #i = 1 if numpy.where(
                     #    abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
                     #ystep = Y[i] / 10.
                         ystep = ystep/5
                         ystep = ystep/(10**digD)
                     else:
                         ystep = ((ymax + (10**(dig)))//10**(dig))*(10**(dig))
                         ystep = ystep/5
                     if self.xaxis is not 'time':
                         dig = int(numpy.log10(xmax))
                         if dig <= 0:
                             digD = len(str(xmax)) - 2
                             xdec = xmax*(10**digD)
                             xstep = ((xdec + (10**(dig)))//10**(dig))*(10**(dig))
                             xstep = xstep*0.5
                             xstep = xstep/(10**digD)
                         else:
                             xstep = ((xmax + (10**(dig)))//10**(dig))*(10**(dig))
                             xstep = xstep/5
                     for n, ax in enumerate(self.axes):
                         if ax.firsttime:
                             ax.set_facecolor(self.bgcolor)
                         if self.save:
                             self.save_figure(n)
                     if self.plot_server:
                         self.send_to_server()
                         # t = Thread(target=self.send_to_server)
                         '{}{}_{}.png'.format(
                             self.CODE,
                             label,
-                            self.getDateTime(self.data.max_time).strftime(
+                            self.getDateTime(self.data.max_time).strftime('%Y%m%d_%H%M%S'),
-                                '%Y%m%d_%H%M%S'
-                                ),
                             )
                         )
                     log.log('Saving figure: {}'.format(figname), self.name)
                     if not os.path.isdir(os.path.dirname(figname)):
                         os.makedirs(os.path.dirname(figname))
                     self.ncols: number of cols
                     self.nplots: number of plots (channels or pairs)
                     self.ylabel: label for Y axes
                     self.titles: list of axes title
                     '''
                     raise NotImplementedError
                     Must be defined in the child class
                     '''
                     raise NotImplementedError
                 def run(self, dataOut, **kwargs):
                     '''
                     Main plotting routine
                     '''
                     if self.isConfig is False:
                         self.__setup(**kwargs)
                         if dataOut.type == 'Parameters':
                             t = dataOut.utctimeInit
                         else:
                             t = dataOut.utctime
                         if dataOut.useLocalTime:
                             self.getDateTime = datetime.datetime.fromtimestamp
                             self.getDateTime = datetime.datetime.utcfromtimestamp
                             if self.localtime:
                                 t -= time.timezone
                         if 'buffer' in self.plot_type:
                             if self.xmin is None:
                                 self.tmin = t
                             else:
                                 self.tmin = (
                                     self.getDateTime(t).replace(
                                         hour=self.xmin,
                                         minute=0,
                                         second=0) - self.getDateTime(0)).total_seconds()
                         self.data.setup()
                     if dataOut.useLocalTime and not self.localtime:
                         tm += time.timezone
                     if self.xaxis is 'time' and self.data and (tm - self.tmin) >= self.xrange*60*60:
                         self.save_counter = self.save_period
                         self.__plot()
                         self.xmin += self.xrange
                         self.__plot()
                     if self.data and self.pause:
                         figpause(10)

schainpy/model/io/jroIO_kamisr.py +205 -185

             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
-            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from numpy import imag
+            @MPDecorator
             class AMISRReader(ProcessingUnit):
                 '''
                 classdocs
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.set = None
                     self.subset = None
                     self.extension_file = '.h5'
                     self.flagIsNewFile = 0
                     self.filename = ''
                     self.amisrFilePointer = None
-                    self.dataset = None
+                    #self.dataset = None
                     self.profileIndex = 0
                     self.beamCodeByFrame = None
                     self.radacTimeByFrame = None
                     self.dataset = None
                     self.__firstFile = True
                     self.buffer = None
                     self.timezone = 'ut'
                     self.__waitForNewFile = 20
                     self.__filename_online = None
                     #Is really necessary create the output object in the initializer
                     self.dataOut = Voltage()
+                    self.dataOut.error=False
                 def setup(self,path=None,
                                 startDate=None,
                                 endDate=None,
                                 startTime=None,
                                 endTime=None,
                                 walk=True,
                                 timezone='ut',
                                 nCode = 0,
                                 nBaud = 0,
                                 online=False):
+                    #print ("T",path)
                     self.timezone = timezone
                     self.all = all
                     self.online = online
                     self.code = code
                     self.nCode = int(nCode)
                     self.nBaud = int(nBaud)
                     #self.findFiles()
                     if not(online):
                         #Busqueda de archivos offline
                         self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
                     else:
                         self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
                     if not(self.filenameList):
                         print("There is no files into the folder: %s"%(path))
                         sys.exit(-1)
                     self.fileIndex = -1
                     self.readNextFile(online)
                     '''
                     Add code
                     '''
                     self.isConfig = True
                     pass
                 def readAMISRHeader(self,fp):
                     header = 'Raw11/Data/RadacHeader'
                     self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
                     self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
                     self.frequency =  fp.get('Rx/Frequency')
                     txAus = fp.get('Raw11/Data/Pulsewidth')
                     self.nblocks = self.pulseCount.shape[0] #nblocks
                     self.nprofiles = self.pulseCount.shape[1] #nprofile
                     self.nsa = self.nsamplesPulse[0,0] #ngates
                     self.nchannels = self.beamCode.shape[1]
                     self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
                     #self.__waitForNewFile = self.nblocks  # wait depending on the number of blocks since each block is 1 sec
                     self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
                     #filling radar controller header parameters
                     self.__ippKm = self.ippSeconds *.15*1e6 # in km
                     self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
                     self.__txB = 0
                     nWindows=1
                     self.__nSamples = self.nsa
                     self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
                     self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
                     #for now until understand why the code saved is different (code included even though code not in tuf file)
                     #self.__codeType = 0
                    # self.__nCode = None
                     self.__nCode = self.nCode
                     self.__nBaud = self.nBaud
                     #self.__code = 0
                     #filling system header parameters
                     self.__nSamples = self.nsa
                     self.newProfiles = self.nprofiles/self.nchannels
                     self.__channelList = list(range(self.nchannels))
                     self.__frequency = self.frequency[0][0]
                 def createBuffers(self):
                     pass
                 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                     self.path = path
                     self.startDate = startDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.walk = walk
                 def __checkPath(self):
                     if os.path.exists(self.path):
                         self.status = 1
                     else:
                         self.status = 0
                         print('Path:%s does not exists'%self.path)
                     return
                 def __selDates(self, amisr_dirname_format):
                     try:
                         year = int(amisr_dirname_format[0:4])
                         month = int(amisr_dirname_format[4:6])
                         dom = int(amisr_dirname_format[6:8])
                         thisDate = datetime.date(year,month,dom)
                         if (thisDate>=self.startDate and thisDate <= self.endDate):
                             return amisr_dirname_format
                     except:
                         return None
                 def __findDataForDates(self,online=False):
                     if not(self.status):
                         return None
                     pat = '\d+.\d+'
                     dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                     dirnameList = [x for x in dirnameList if x!=None]
                     else:
                         self.status = 0
                         return None
                 def __getTimeFromData(self):
                     startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                     endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                         filename = self.filenameList[i]
                         fp = h5py.File(filename,'r')
                         time_str = fp.get('Time/RadacTimeString')
-                        startDateTimeStr_File = time_str[0][0].split('.')[0]
+                        startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
+                        #startDateTimeStr_File = "2019-12-16 09:21:11"
                         junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
-                        endDateTimeStr_File = time_str[-1][-1].split('.')[0]
+                        #endDateTimeStr_File = "2019-12-16 11:10:11"
+                        endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
                         junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                         endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                         fp.close()
+                        #print("check time", startDateTime_File)
                         if self.timezone == 'lt':
                             startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                             endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
                         if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
                             #self.filenameList.remove(filename)
                             filter_filenameList.append(filename)
                         if (endDateTime_File>=endDateTime_Reader):
                             break
                     filter_filenameList.sort()
                     self.filenameList = filter_filenameList
                     return 1
                 def __filterByGlob1(self, dirName):
                     filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
                     filter_files.sort()
                     filterDict.setdefault(dirName)
                     filterDict[dirName] = filter_files
                     return filterDict
                 def __getFilenameList(self, fileListInKeys, dirList):
                     for value in fileListInKeys:
                         dirName = list(value.keys())[0]
                         for file in value[dirName]:
                             filename = os.path.join(dirName, file)
                             self.filenameList.append(filename)
                 def __selectDataForTimes(self, online=False):
                     #aun no esta implementado el filtro for tiempo
                     if not(self.status):
                         return None
                     dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                     fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
                     self.__getFilenameList(fileListInKeys, dirList)
                     if not(online):
                         #filtro por tiempo
                         else:
                             self.status = 0
                             return None
                     else:
                         #get the last file - 1
                         self.filenameList = [self.filenameList[-2]]
                     new_dirnameList = []
                     for dirname in self.dirnameList:
                         junk = numpy.array([dirname in x for x in self.filenameList])
                             new_dirnameList.append(dirname)
                     self.dirnameList = new_dirnameList
                     return 1
                 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),walk=True):
                     if endDate ==None:
                      startDate = datetime.datetime.utcnow().date()
                      endDate = datetime.datetime.utcnow().date()
                     self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
                     self.__checkPath()
                     self.__findDataForDates(online=True)
                     self.dirnameList = [self.dirnameList[-1]]
                     self.__selectDataForTimes(online=True)
                     return
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         walk=True):
                     self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                     self.__checkPath()
                     self.__findDataForDates()
                     self.__selectDataForTimes()
                     for i in range(len(self.filenameList)):
                         print("%s" %(self.filenameList[i]))
                     return
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
+                            self.dataOut.error = True
                             return 0
                         filename = self.filenameList[idFile]
                         amisrFilePointer = h5py.File(filename,'r')
                         break
                     self.flagIsNewFile = 1
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setNextFileOnline(self):
                     filename = self.filenameList[0]
                     if self.__filename_online != None:
                             self.__selectDataForTimes(online=True)
                             filename = self.filenameList[0]
                             wait += 1
                     self.__filename_online = filename
                     self.amisrFilePointer = h5py.File(filename,'r')
                     self.flagIsNewFile = 1
                     self.filename = filename
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def readData(self):
                     buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
                     re = buffer[:,:,:,0]
                     im = buffer[:,:,:,1]
                     dataset = re + im*1j
                     self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
                     timeset = self.radacTime[:,0]
                     return dataset,timeset
                 def reshapeData(self):
                 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
                     channels = self.beamCodeByPulse[0,:]
                     nchan = self.nchannels
                     #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
                     nblocks = self.nblocks
                     nsamples = self.nsa
                     #Dimensions : nChannels, nProfiles, nSamples
-                    new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
+                    new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
                     ############################################
                     for thisChannel in range(nchan):
                         new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
                     new_block = numpy.transpose(new_block, (1,0,2,3))
                     new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
                     return new_block
                 def updateIndexes(self):
                     pass
                 def fillJROHeader(self):
                     #fill radar controller header
-                    self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
+                    self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
                                                                                   txA=self.__txA,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code,
                                                                                   fClock=1)
                     #fill system header
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=self.newProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14,
-                                                                pciDioBusWith=32)
+                                                                pciDioBusWidth=32)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = self.newProfiles*self.nblocks
                     #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     ranges = numpy.reshape(self.rangeFromFile.value,(-1))
                     self.dataOut.heightList =  ranges/1000.0 #km
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     #self.dataOut.timeZone = -5 #self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     if self.timezone == 'lt':
                         self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
                     else:
                         self.dataOut.timeZone = 0 #by default time is UTC
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = self.ippSeconds
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.online
                     pass
                 def readNextFile(self,online=False):
                     if not(online):
                         newFile = self.__setNextFileOffline()
                     else:
                         newFile = self.__setNextFileOnline()
                     if not(newFile):
                         return 0
                     #if self.__firstFile:
                     self.readAMISRHeader(self.amisrFilePointer)
                     self.createBuffers()
                     self.fillJROHeader()
                     #self.__firstFile = False
                     self.dataset,self.timeset = self.readData()
                     if self.endDate!=None:
                      endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                      time_str = self.amisrFilePointer.get('Time/RadacTimeString')
-                     startDateTimeStr_File = time_str[0][0].split('.')[0]
+                     startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
                      junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                      startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                      if self.timezone == 'lt':
                       startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                      if (startDateTime_File>endDateTime_Reader):
                          return 0
                     self.jrodataset = self.reshapeData()
                     #----self.updateIndexes()
                     self.profileIndex = 0
                     return 1
                 def __hasNotDataInBuffer(self):
                     if self.profileIndex >= (self.newProfiles*self.nblocks):
                         return 1
                     return 0
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return 0
                     if self.__hasNotDataInBuffer():
                         if not (self.readNextFile(self.online)):
                             return 0
-                    if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
+                    if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
                         self.dataOut.flagNoData = True
                         return 0
                     #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
                     self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
+                    #print("R_t",self.timeset)
                     #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
                     #verificar basic header de jro data y ver si es compatible con este valor
                     #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
                     indexprof = numpy.mod(self.profileIndex, self.newProfiles)
                     indexblock = self.profileIndex/self.newProfiles
-                    #print indexblock, indexprof
+                    #print (indexblock, indexprof)
-                    self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
+                    diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
+                    diffUTC = 0
+                    t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
+                    #cambio posible 18/02/2020
+                    #print("utc :",indexblock," __ ",t_comp)
+                    #print(numpy.shape(self.timeset))
+                    self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
+                    #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
+                    #print(self.dataOut.utctime)
                     self.dataOut.profileIndex = self.profileIndex
                     self.dataOut.flagNoData = False
             #         if indexprof == 0:
             #             print self.dataOut.utctime
                     self.profileIndex += 1
                     return self.dataOut.data
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()

schainpy/model/io/jroIO_param.py +36 -36

                     except IOError:
                         traceback.print_exc()
                         raise IOError("The file %s can't be opened" %(filename))
                     #In case has utctime attribute
                     grp2 = grp1['utctime']
              #         thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
                 setType = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                         dsDict['variable'] = self.dataList[i]
                         #---------------------    Conditionals    ------------------------
                         #There is no data
                         if dataAux is None:
                             return 0
                         if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                         return False
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     if os.path.exists(fullpath):
                         filesList = os.listdir( fullpath )
-                        filesList = [k for k in filesList if 'M' in k]
+                        ##filesList = [k for k in filesList if 'M' in k]
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[i]
                                 tableName = dsInfo['dsName']
                                 if dsInfo['nDim'] == 3:
                                     shape = dsInfo['shape'].astype(int)
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList, mode=mode,
                                    setType=setType)
                     self.putData()
                     return
             @MPDecorator
             class ParameterReader(Reader, ProcessingUnit):
                     self.set_kwargs(**kwargs)
                     if not self.ext.startswith('.'):
                         self.ext = '.{}'.format(self.ext)
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                         pathname, filename = os.path.split(fullpath)
                         self.year = int(filename[1:5])
                         self.doy = int(filename[5:8])
                         self.set = int(filename[8:11]) - 1
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                     return
                 def readFirstHeader(self):
                     '''Read metadata and data'''
                     self.__readMetadata()
                     self.__readData()
                     self.__setBlockList()
                     self.blockIndex = 0
                     return
                 def __setBlockList(self):
                             else:
                                 data = gp[name].value
                                 listMetaname.append(name)
                                 listMetadata.append(data)
                     elif self.metadata:
                         metadata = json.loads(self.metadata)
                         listShapes = {}
                     self.listShapes = listShapes
                     self.listMetaname = listMetaname
                     self.listMeta = listMetadata
                     return
                     listdataname = []
                     listdata = []
                     if 'Data' in self.fp:
                         grp = self.fp['Data']
                         for item in list(grp.items()):
                                 for i in range(dim):
                                     array.append(grp[name]['table{:02d}'.format(i)].value)
                                 array = numpy.array(array)
                             listdata.append(array)
                     elif self.metadata:
                         metadata = json.loads(self.metadata)
                     self.listDataname = listdataname
                     self.listData = listdata
                     return
                 def getData(self):
                     for i in range(len(self.listMeta)):
                 lastTime = None
                 def __init__(self):
                     Operation.__init__(self)
                     return
                             dsDict['nDim'] = len(dataAux.shape)
                             dsDict['shape'] = dataAux.shape
                             dsDict['dsNumber'] = dataAux.shape[0]
                         dsList.append(dsDict)
                         tableList.append((self.dataList[i], dsDict['nDim']))
                         self.lastTime = currentTime
                         self.currentDay = dataDay
                         return False
                     timeDiff = currentTime - self.lastTime
                     #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                     self.dataOut = dataOut
                     if not(self.isConfig):
                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                    metadataList=metadataList, dataList=dataList,
                                    setType=setType)
                     self.putData()
                     return
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     setFile = self.setFile
                     return
                 def writeData(self, fp):
                     grp = fp.create_group("Data")
                     dtsets = []
                     data = []
                     for dsInfo in self.dsList:
                         if dsInfo['nDim'] == 0:
                             ds = grp.create_dataset(
                                 dsInfo['variable'],
                                 (self.blocksPerFile, ),
                                 chunks=True,
                                 dtype=numpy.float64)
                             dtsets.append(ds)
                             data.append((dsInfo['variable'], -1))
                             sgrp = grp.create_group(dsInfo['variable'])
                             for i in range(dsInfo['dsNumber']):
                                 ds = sgrp.create_dataset(
                                     'table{:02d}'.format(i),
                                     (self.blocksPerFile, ) + dsInfo['shape'][1:],
                                     chunks=True)
                                 dtsets.append(ds)
                     fp.flush()
                     log.log('Creating file: {}'.format(fp.filename), self.name)
                     self.ds = dtsets
                     self.data = data
                     self.firsttime = True

schainpy/model/proc/jroproc_base.py +37 -40

             Jan 2018
             Abstract:
                 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
                 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
                 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
             Based on:
                 $Author: murco $
                 """
                 Update - Jan 2018 - MULTIPROCESSING
                 All the "call" methods present in the previous base were removed.
                 The majority of operations are independant processes, thus
                 the decorator is in charge of communicate the operation processes
                 with the proccessing unit via IPC.
                 The constructor does not receive any argument. The remaining methods
                 are related with the operations to execute.
                 """
                 proc_type = 'processing'
                 def addOperation(self, conf, operation):
                     """
                     This method is used in the controller, and update the dictionary containing the operations to execute. The dict
                     posses the id of the operation process (IPC purposes)
                         Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
                     self.operations.append(
                         (operation, conf.type, conf.id, conf.getKwargs()))
                     if 'plot' in self.name.lower():
                         self.plots.append(operation.CODE)
                     return
             class InputQueue(Thread):
                 '''
                 Class to hold input data for Proccessing Units and external Operations,
                 '''
                 def get(self):
                     if not self.islocked and self.size/1000000 > 512:
                         self.lock.n.value += 1
                         self.islocked = True
                         self.lock.clear()
                     elif self.islocked and self.size/1000000 <= 512:
                         self.islocked = False
                         self.lock.n.value -= 1
                         if self.lock.n.value == 0:
                             self.lock.set()
                     obj = self.queue.get()
                     self.size -= sys.getsizeof(obj)
                     return pickle.loads(obj)
             def MPDecorator(BaseClass):
                 """
                 Multiprocessing class decorator
                 This function add multiprocessing features to a BaseClass. Also, it handle
                 the communication beetween processes (readers, procUnits and operations).
                 """
                 class MPClass(BaseClass, Process):
                         self.t = time.time()
                         self.name = BaseClass.__name__
                         self.__doc__ = BaseClass.__doc__
                         if 'plot' in self.name.lower() and not self.name.endswith('_'):
                             self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
                         self.start_time = time.time()
                         self.id = args[0]
                         self.inputId = args[1]
                         self.project_id = args[2]
                         '''
                         self.queue.start()
                     def listen(self):
                         '''
                         This function waits for objects
                         '''
                         return self.queue.get()
                     def set_publisher(self):
                         '''
                         This function create a zmq socket for publishing objects.
                         '''
                         time.sleep(0.5)
                         c = zmq.Context()
                         self.sender = c.socket(zmq.PUB)
                         self.sender.connect(
                         '''
                         This function publish an object, to an specific topic.
                         It blocks publishing when receiver queue is full to avoid data loss
                         '''
                         if self.inputId is None:
                             self.lock.wait()
                         self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
                     def runReader(self):
                         '''
                         Run fuction for read units
                             try:
                                 BaseClass.run(self, **self.kwargs)
                             except:
                                 err = traceback.format_exc()
                                 if 'No more files' in err:
                                     log.warning('No more files to read', self.name)
                                 else:
                                     self.err_queue.put('{}|{}'.format(self.name, err))
                                 self.dataOut.error = True
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'self' and not self.dataOut.flagNoData:
                                     op(**kwargs)
                                 continue
                             self.publish(self.dataOut, self.id)
+                            if self.dataOut.error:
-                            if self.dataOut.error:
                                 break
                         time.sleep(0.5)
                         '''
                         while True:
                             self.dataIn = self.listen()
                             if self.dataIn.flagNoData and self.dataIn.error is None:
                                 continue
                             elif self.dataIn.error:
                                 self.dataOut.error = self.dataIn.error
                                 self.dataOut.flagNoData = True
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'self' and not self.dataOut.flagNoData:
                                     op(**kwargs)
                                 elif optype == 'other' and not self.dataOut.flagNoData:
                                     self.dataOut = op.run(self.dataOut, **kwargs)
                                 elif optype == 'external' and not self.dataOut.flagNoData:
                                     self.publish(self.dataOut, opId)
                             self.publish(self.dataOut, self.id)
                             for op, optype, opId, kwargs in self.operations:
                                 if optype == 'external' and self.dataOut.error:
                                     self.publish(self.dataOut, opId)
                             if self.dataOut.error:
                                 break
                         time.sleep(0.5)
                     def runOp(self):
                         Run function for external operations (this operations just receive data
                         ex: plots, writers, publishers)
                         '''
                         while True:
                             dataOut = self.listen()
                                     self.err_queue.put('{}|{}'.format(self.name, traceback.format_exc()))
                                     dataOut.error = True
                             else:
                                 break
                     def run(self):
                         if self.typeProc is "ProcUnit":
                             if self.inputId is not None:
                                 self.subscribe()
                             self.set_publisher()
                             if 'Reader' not in BaseClass.__name__:
                                 self.runProc()
                             else:
                                 self.runReader()
                         elif self.typeProc is "Operation":
                             self.subscribe()

schainpy/model/proc/jroproc_parameters.py 0 0

	1	NO CONTENT: modified file			NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_spectra.py +31 -31

                     # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
                     self.dataOut.channelList = range(len(channelIndexList))
                     self.__selectPairsByChannel(channelIndexList)
                     return 1
                 def selectFFTs(self, minFFT, maxFFT ):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
                     minFFT<= FFT <= maxFFT
                     """
                     if (minFFT > maxFFT):
                         raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
                     self.selectFFTsByIndex(minIndex, maxIndex)
                     return 1
                 def setH0(self, h0, deltaHeight = None):
                     if not deltaHeight:
                         deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     nHeights = self.dataOut.nHeights
                     newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
                     self.dataOut.heightList = newHeiRange
                 def selectHeights(self, minHei, maxHei):
                     """
                     Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
 si el metodo se ejecuto con exito caso contrario devuelve 0
                     """
                     if (minHei > maxHei):
                         raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minHei, maxHei))
                         maxIndex = len(heights)
                     self.selectHeightsByIndex(minIndex, maxIndex)
                     return 1
                 def selectFFTsByIndex(self, minIndex, maxIndex):
                     """
                     """
                     if (minIndex < 0) or (minIndex > maxIndex):
                     self.dataOut.data_spc = data_spc
                     self.dataOut.data_cspc = data_cspc
                     self.dataOut.data_dc = data_dc
                     self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
                     self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
                     self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
                         xx_inv = numpy.linalg.inv(xx)
                         xx_aux = xx_inv[0, :]
                         for ich in range(num_chan):
                             yy = jspectra[ich, ind_vel, :]
                             jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
                     return 1
                 def removeInterference2(self):
                     cspc = self.dataOut.data_cspc
                     spc = self.dataOut.data_spc
                     Heights = numpy.arange(cspc.shape[2])
                     realCspc = numpy.abs(cspc)
                     for i in range(cspc.shape[0]):
                         LinePower= numpy.sum(realCspc[i], axis=0)
                         Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
                         InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
                         InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
                         InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
                         InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
                         #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
                         if len(InterferenceRange)<int(cspc.shape[1]*0.3):
                             cspc[i,InterferenceRange,:] = numpy.NaN
                     self.dataOut.data_cspc = cspc
                 def removeInterference(self,  interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None):
                     jspectra = self.dataOut.data_spc
                     if n is not None:
                         self.n = int(n)
                     else:
                         self.__integrationtime = int(timeInterval)
                         self.n = None
                         self.__byTime = True
                 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
                     if n == 1:
                         return
                     dataOut.flagNoData = True
                     if not self.isConfig:
                         dataOut.data_spc = avgdata_spc
                         dataOut.data_cspc = avgdata_cspc
                         dataOut.data_dc = avgdata_dc
                         dataOut.nIncohInt *= self.n
                         dataOut.utctime = avgdatatime
                         dataOut.flagNoData = False
-                    return dataOut
  No newline at end of file
+                    return dataOut

schainpy/model/proc/jroproc_voltage.py +20 -21

             @MPDecorator
             class VoltageProc(ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut.data = data
                     # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
                     self.dataOut.channelList = range(len(channelIndexList))
                     return 1
                 def selectHeights(self, minHei=None, maxHei=None):
                         """
                         Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                         """
                         buffer = self.dataOut.data[:, :, 0:int(self.dataOut.nHeights-r)]
                         buffer = buffer.reshape(self.dataOut.nChannels, self.dataOut.nProfiles, int(self.dataOut.nHeights/window), window)
                         buffer = numpy.sum(buffer,3)
                         # print self.__bufferStride[self.__profIndexStride - 1]
                         # raise
                         return self.__bufferStride[self.__profIndexStride - 1]
                     return None, None
                 def integrate(self, data, datatime=None):
                     avgdatatime = self.__initime
                     deltatime = datatime - self.__lastdatatime
                     if not self.__withOverlapping:
                         self.__initime = datatime
                     else:
                     avgdatatime = (times - 1) * timeInterval + dataOut.utctime
                     self.__dataReady = True
                     return avgdata, avgdatatime
                 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
                     if not self.isConfig:
                         avgdata, avgdatatime = self.integrateByBlock(dataOut)
                         dataOut.nProfiles /= self.n
                     else:
                         if stride is None:
                             avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
                         else:
                             avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
                     #   dataOut.timeInterval *= n
                     dataOut.flagNoData = True
                     self.nCode = len(code)
                     self.nBaud = len(code[0])
                     if (osamp != None) and (osamp >1):
                         self.osamp = osamp
                         self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
                     #Frequency
                     __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
                     __codeBuffer[:,0:self.nBaud] = self.code
                     self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
                     junk = junk.flatten()
                     code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
                     profilesList = range(self.__nProfiles)
                     for i in range(self.__nChannels):
                         for j in profilesList:
                             self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
                     return self.datadecTime
                 def __convolutionByBlockInFreq(self, data):
                     return data
                 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
                     if dataOut.flagDecodeData:
                     self.__nProfiles = dataOut.nProfiles
                     datadec = None
                     if mode == 3:
                         mode = 0
                         if shape[2] % n != 0:
                             raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
                         new_shape = shape[0], shape[1]*n, int(shape[2]/n)
                         dataOut.data = numpy.reshape(dataOut.data, new_shape)
                         dataOut.flagNoData = False

schainpy/model/utils/jroutils_ftp.py +27 -27

                     self.mutex.acquire()
                     #         init = time.time()
+                    #
                     #         while(self.bussy):
                     #             sleep(0.1)
                     #             if time.time() - init > 2*self.period:
                     #                 return 0
                     self.fileList = fileList
                     self.mutex.release()
                     return 1
                         if self.stopFlag:
                             break
                         #   self.bussy = True
                         self.mutex.acquire()
                     """
                         This method is used to set SSH parameters and establish a connection to a remote server
                         Inputs:
                             server    - remote server IP Address
                             username    - remote server Username
                             password    - remote server password
                             remotefolder    - remote server current working directory
                         Return: void
                         Affects:
                             self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                 def __execute(self, command):
                     """
                         __execute a command on remote server
                         Input:
                             command    - Exmaple 'ls -l'
                         Return:
 in error case else 1
                     """
                 def mkdir(self, remotefolder):
                     """
                         mkdir is used to make a new directory in remote server
                         Input:
                             remotefolder    - directory name
                         Return:
 in error case else 1
                     """
                 def cd(self, remotefolder):
                     """
                         cd is used to change remote working directory on server
                         Input:
                             remotefolder    - current working directory
                         Affects:
                             self.remotefolder
                         Return:
 in case of error else 1
                     """
                     if not self.status:
                     ProcessingUnit.__init__(self, **kwargs)
                     self.isConfig = False
                     self.clientObj = None
                 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
                     self.clientObj = None
                         self.init = time.time()
                         self.setup(**kwargs)
                         self.isConfig = True
                     if not self.clientObj.is_alive():
                         print("[Remote Server]: Restarting connection ")
                         self.setup(**kwargs)
                     if time.time() - self.init >= self.period:
                         fullfilenameList = self.findFiles()
                     try:
                         self.ftp = ftplib.FTP(self.server)
                         self.ftp.login(self.username,self.password)
                         self.ftp.cwd(self.remotefolder)
                         #   print 'Connect to FTP Server: Successfully'
                     except ftplib.all_errors:
                         print('Error FTP Service')
                         self.status = 1
                         self.counter = 0
-                    self.status = 1
  No newline at end of file
+                    self.status = 1

schainpy/model/utils/jroutils_publish.py +20 -20

             def get_plot_code(s):
                 label = s.split('_')[0]
                 codes = [key for key in PLOT_CODES if key in label]
                 if codes:
                     return PLOT_CODES[codes[0]]
                 else:
                     return 24
                     self.counter = 0
                     self.delay = kwargs.get('delay', 0)
                     self.cnt = 0
                     self.verbose = verbose
                     context = zmq.Context()
                     self.zmq_socket = context.socket(zmq.PUSH)
                     server = kwargs.get('server', 'zmq.pipe')
                 def publish_data(self):
                     self.dataOut.finished = False
                     if self.verbose:
                         log.log(
                             'Sending {} - {}'.format(self.dataOut.type, self.dataOut.datatime),
                     time.sleep(self.delay)
                 def close(self):
                     self.dataOut.finished = True
                     self.zmq_socket.send_pyobj(self.dataOut)
                     time.sleep(0.1)
                     self.zmq_socket.close()
             class ReceiverData(ProcessingUnit):
                             self.ftp.close()
                         self.ftp = None
                         self.ready = False
                         return
                     try:
                         self.ftp.login(self.username, self.password)
                 def upload(self, src, dst):
                     log.log('Uploading {} -> {} '.format(
                         src.split('/')[-1], dst.split('/')[-1]),
                         self.name,
                         nl=False
                         )
                     fp.close()
                     log.success('OK', tag='')
                     return 1
                 def send_files(self):
                     for x, pattern in enumerate(self.patterns):
                             srcname = self.find_files(local, ext)
                             src = os.path.join(local, srcname)
                             if os.path.getmtime(src) < time.time() - 30*60:
                                 log.warning('Skipping old file {}'.format(srcname))
                                 continue
                             if srcname is None or srcname == self.latest[x]:
                                 log.warning('File alreday uploaded {}'.format(srcname))
                                 continue
                             if 'png' in ext:
                                 dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code))
                             else:
                                 dstname = srcname
                             dst = os.path.join(remote, dstname)
                             if self.upload(src, dst):
                                 self.times[x] = time.time()
                                 self.latest[x] = srcname
                             else:
                                 self.ready = False
                                 break
                 def run(self, dataOut, server, username, password, timeout=10, **kwargs):
                     if not self.isConfig:
                         self.setup(
                             server=server,
                             username=username,
                             password=password,
                             timeout=timeout,
                             **kwargs
                             )
                         self.isConfig = True

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages