schain Commit - r1279:c53fe2a4a291 · Jicamarca Repository

Modificación a kmamisr para ejecutarse en la versión 3, creación de scripts con terminación v3 para difereciarlos, se comentó la linea #720 de JroIO_param.py debido a que reiniciaba la lista de archivos, ocasionando la reescritura del archivo hdf5. Alguna otra modificación aparente es producto de algunas variaciones en espacios al usar la función print()

joabAM -

r1279:c53fe2a4a291

parent child

Context file:

r1279:c53fe2a4a291

Collapse all files

The requested changes are too big and content was truncated. Show full diff

schainpy/controller.py +41 -36

                      nFiles = len(files)
                      if nFiles == 0:
                          continue
-                     skip = int(math.ceil(nFiles / n))
+                     skip = int(math.ceil(nFiles / n))
                      while nFiles > cursor * skip:
                          rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
                                       skip=skip)
                      time.sleep(3)
              def wait(context):
                  time.sleep(1)
                  c = zmq.Context()
                  receiver = c.socket(zmq.SUB)
-                 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
+                 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
                  receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
                  msg = receiver.recv_multipart()[1]
                  context.terminate()
                          parmElement.set('name', self.name)
                          parmElement.set('value', self.value)
                          parmElement.set('format', self.format)
                  def readXml(self, parmElement):
                      self.id = parmElement.get('id')
                      self.name = opElement.get('name')
                      self.type = opElement.get('type')
                      self.priority = opElement.get('priority')
-                     self.project_id = str(project_id)
+                     self.project_id = str(project_id)
                      # Compatible with old signal chain version
                      # Use of 'run' method instead 'init'
                      self.id = None
                      self.datatype = None
                      self.name = None
-                     self.inputId = None
+                     self.inputId = None
                      self.opConfObjList = []
                      self.procUnitObj = None
                      self.opObjDict = {}
                      return self.id
-                 def updateId(self, new_id):
+                 def updateId(self, new_id):
                      '''
                      new_id = int(parentId) * 10 + (int(self.id) % 10)
                      new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
                          id sera el topico a publicar
                          inputId sera el topico a subscribirse
                      '''
                      # Compatible with old signal chain version
                      if datatype == None and name == None:
                          raise ValueError('datatype or name should be defined')
                      self.lock = lock
                      self.opConfObjList = []
-                     self.addOperation(name='run', optype='self')
+                     self.addOperation(name='run', optype='self')
                  def removeOperations(self):
                      '''
                      className = eval(self.name)
+                     #print(self.name)
                      kwargs = self.getKwargs()
+                     #print(kwargs)
+                     #print("mark_a")
                      procUnitObj = className(self.id, self.inputId, self.project_id, self.err_queue, self.lock, 'ProcUnit', **kwargs)
+                     #print("mark_b")
                      log.success('creating process...', self.name)
                      for opConfObj in self.opConfObjList:
                          if opConfObj.type == 'self' and opConfObj.name == 'run':
                              continue
                          elif opConfObj.type == 'self':
                              opObj = getattr(procUnitObj, opConfObj.name)
                          else:
                              opObj = opConfObj.createObject()
                          log.success('adding operation: {}, type:{}'.format(
                              opConfObj.name,
                              opConfObj.type), self.name)
                          procUnitObj.addOperation(opConfObj, opObj)
                      procUnitObj.start()
                      self.procUnitObj = procUnitObj
                  def close(self):
                      for opConfObj in self.opConfObjList:
                  def getElementName(self):
-                     return self.ELEMENTNAME
+                     return self.ELEMENTNAME
                  def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
                            startTime='', endTime='', server=None, **kwargs):
                      kwargs deben ser trasmitidos en la instanciacion
                      '''
                      # Compatible with old signal chain version
                      if datatype == None and name == None:
                          raise ValueError('datatype or name should be defined')
                      self.datatype = datatype
                      if path != '':
                          self.path = os.path.abspath(path)
+                     print (self.path)
                      self.startDate = startDate
                      self.endDate = endDate
                      self.startTime = startTime
                      self.endTime = endTime
                      self.server = server
-                     self.err_queue = err_queue
+                     self.err_queue = err_queue
                      self.addRunOperation(**kwargs)
                  def update(self, **kwargs):
                  def addRunOperation(self, **kwargs):
-                     opObj = self.addOperation(name='run', optype='self')
+                     opObj = self.addOperation(name='run', optype='self')
                      if self.server is None:
                          opObj.addParameter(
                      print('*' * 19)
                      print(' ')
                      self.id = str(id)
-                     self.description = description
+                     self.description = description
                      self.email = email
                      self.alarm = alarm
                      if name:
                      readUnitConfObj = ReadUnitConf()
                      readUnitConfObj.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
                      self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
                      return readUnitConfObj
                  def addProcUnit(self, inputId='0', datatype=None, name=None):
                      idProcUnit = self.__getNewId()
                      procUnitConfObj = ProcUnitConf()
-                     input_proc = self.procUnitConfObjDict[inputId]
+                     input_proc = self.procUnitConfObjDict[inputId]
                      procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue, input_proc.lock)
                      self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
                      t = Thread(target=self.__monitor, args=(self.err_queue, self.ctx))
                      t.start()
                  def __monitor(self, queue, ctx):
                      import socket
                      procs = 0
                      err_msg = ''
                      while True:
                          msg = queue.get()
                          if '#_start_#' in msg:
                              procs -=1
                          else:
                              err_msg = msg
-                         if procs == 0 or 'Traceback' in err_msg:
+                         if procs == 0 or 'Traceback' in err_msg:
                              break
                          time.sleep(0.1)
                      if '|' in err_msg:
                          name, err = err_msg.split('|')
                          if 'SchainWarning' in err:
                              log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
                          else:
                              log.error(err, name)
-                     else:
+                     else:
                          name, err = self.name, err_msg
                      time.sleep(2)
                      for conf in self.procUnitConfObjDict.values():
                              if confop.type == 'external':
                                  confop.opObj.terminate()
                          conf.procUnitObj.terminate()
                      ctx.term()
                      message = ''.join(err)
                              subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
                          a = Alarm(
-                             modes=self.alarm,
+                             modes=self.alarm,
                              email=self.email,
                              message=message,
                              subject=subject,
                      if not os.path.exists('/tmp/schain'):
                          os.mkdir('/tmp/schain')
                      self.ctx = zmq.Context()
                      xpub = self.ctx.socket(zmq.XPUB)
                      xpub.bind('ipc:///tmp/schain/{}_pub'.format(self.id))
                  def run(self):
                      log.success('Starting {}: {}'.format(self.name, self.id), tag='')
-                     self.start_time = time.time()
-                     self.createObjects()
-                     self.setProxy()
+                     self.start_time = time.time()
+                     self.createObjects()
+                     self.setProxy()
                      log.success('{} Done (Time: {}s)'.format(
                          self.name,
                          time.time()-self.start_time), '')

schainpy/model/data/jrodata.py +13 -13

                  flagNoData = True
                  def copy(self, inputObj=None):
                      if inputObj == None:
                          return copy.deepcopy(self)
                      deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
                      velrange = deltav * (numpy.arange(self.nFFTPoints + extrapoints) - self.nFFTPoints / 2.)
                      if self.nmodes:
                          return velrange/self.nmodes
                      else:
                  MAXNUMY = 100
                  def __init__(self, code, throttle_value, exp_code, buffering=True, snr=False):
                      self.key = code
                      self.throttle = throttle_value
                      self.exp_code = exp_code
                      return len(self.__times)
                  def __getitem__(self, key):
                      if key not in self.data:
                          raise KeyError(log.error('Missing key: {}'.format(key)))
                      if 'spc' in key or not self.buffering:
                          elif 'spc_moments' == plot:
                              plot = 'moments'
                          self.data[plot] = {}
                      if 'spc' in self.data or 'rti' in self.data or 'cspc' in self.data or 'moments' in self.data:
                          self.data['noise'] = {}
                          self.data['rti'] = {}
                              self.plottypes.append('noise')
                          if 'rti' not in self.plottypes:
                              self.plottypes.append('rti')
                  def shape(self, key):
                      '''
                      Get the shape of the one-element data for the given key
                      '''
                      Update data object with new dataOut
                      '''
                      if tm in self.__times:
                          return
                      self.profileIndex = dataOut.profileIndex
                      self.tm = tm
                      self.type = dataOut.type
                      self.parameters = getattr(dataOut, 'parameters', [])
                      if hasattr(dataOut, 'meta'):
                          self.meta.update(dataOut.meta)
                      self.pairs = dataOut.pairsList
                      self.interval = dataOut.getTimeInterval()
                      self.localtime = dataOut.useLocalTime
                      self.__heights.append(dataOut.heightList)
                      self.__all_heights.update(dataOut.heightList)
                      self.__times.append(tm)
                      for plot in self.plottypes:
                          if plot in ('spc', 'spc_moments'):
                              z = dataOut.data_spc/dataOut.normFactor
                          if plot == 'scope':
                              buffer = dataOut.data
                              self.flagDataAsBlock = dataOut.flagDataAsBlock
-                             self.nProfiles = dataOut.nProfiles
+                             self.nProfiles = dataOut.nProfiles
                          if plot == 'spc':
                              self.data['spc'] = buffer
                          elif plot == 'cspc':
                      else:
                          meta['xrange'] = []
-                     meta.update(self.meta)
+                     meta.update(self.meta)
                      ret['metadata'] = meta
                      return json.dumps(ret)

schainpy/model/data/jroheaderIO.py +2 -2

                  structure = SYSTEM_STRUCTURE
                  def __init__(self, nSamples=0, nProfiles=0, nChannels=0, adcResolution=14, pciDioBusWidth=0):
                      self.size = 24
                      self.nSamples = nSamples
                      self.nProfiles = nProfiles
              def get_dtype_width(index):
-                 return DTYPE_WIDTH[index]
  No newline at end of file
+                 return DTYPE_WIDTH[index]

schainpy/model/graphics/jroplot_base.py +23 -25

                      self.__throttle_plot = apply_throttle(self.throttle)
                      self.data = PlotterData(
                          self.CODE, self.throttle, self.exp_code, self.buffering, snr=self.showSNR)
                      if self.plot_server:
                          if not self.plot_server.startswith('tcp://'):
                              self.plot_server = 'tcp://{}'.format(self.plot_server)
                      self.setup()
-                     self.time_label = 'LT' if self.localtime else 'UTC'
+                     self.time_label = 'LT' if self.localtime else 'UTC'
                      if self.width is None:
                          self.width = 8
                              cmap = plt.get_cmap(self.colormap)
                          cmap.set_bad(self.bgcolor, 1.)
                          self.cmaps.append(cmap)
                      for fig in self.figures:
                          fig.canvas.mpl_connect('key_press_event', self.OnKeyPress)
                          fig.canvas.mpl_connect('scroll_event', self.OnBtnScroll)
                                  xmax += time.timezone
                          else:
                              xmax = self.xmax
                      ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
                      ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
                      #Y = numpy.array([1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 50000])
                      #i = 1 if numpy.where(
                      #    abs(ymax-ymin) <= Y)[0][0] < 0 else numpy.where(abs(ymax-ymin) <= Y)[0][0]
                      #ystep = Y[i] / 10.
                          ystep = ystep/5
                          ystep = ystep/(10**digD)
-                     else:
+                     else:
                          ystep = ((ymax + (10**(dig)))//10**(dig))*(10**(dig))
                          ystep = ystep/5
                      if self.xaxis is not 'time':
                          dig = int(numpy.log10(xmax))
                          if dig <= 0:
                              digD = len(str(xmax)) - 2
                              xdec = xmax*(10**digD)
                              xstep = ((xdec + (10**(dig)))//10**(dig))*(10**(dig))
                              xstep = xstep*0.5
                              xstep = xstep/(10**digD)
-                         else:
+                         else:
                              xstep = ((xmax + (10**(dig)))//10**(dig))*(10**(dig))
                              xstep = xstep/5
                      for n, ax in enumerate(self.axes):
                          if ax.firsttime:
                              ax.set_facecolor(self.bgcolor)
                          if self.save:
                              self.save_figure(n)
                      if self.plot_server:
                          self.send_to_server()
                          # t = Thread(target=self.send_to_server)
                          '{}{}_{}.png'.format(
                              self.CODE,
                              label,
-                             self.getDateTime(self.data.max_time).strftime(
-                                 '%Y%m%d_%H%M%S'
-                                 ),
+                             self.getDateTime(self.data.max_time).strftime('%Y%m%d_%H%M%S'),
                              )
                          )
                      log.log('Saving figure: {}'.format(figname), self.name)
                      if not os.path.isdir(os.path.dirname(figname)):
                          os.makedirs(os.path.dirname(figname))
                      self.ncols: number of cols
                      self.nplots: number of plots (channels or pairs)
                      self.ylabel: label for Y axes
-                     self.titles: list of axes title
+                     self.titles: list of axes title
                      '''
                      raise NotImplementedError
                      Must be defined in the child class
                      '''
                      raise NotImplementedError
                  def run(self, dataOut, **kwargs):
                      '''
                      Main plotting routine
                      '''
                      if self.isConfig is False:
                          self.__setup(**kwargs)
                          if dataOut.type == 'Parameters':
                              t = dataOut.utctimeInit
                          else:
-                             t = dataOut.utctime
+                             t = dataOut.utctime
                          if dataOut.useLocalTime:
                              self.getDateTime = datetime.datetime.fromtimestamp
                              self.getDateTime = datetime.datetime.utcfromtimestamp
                              if self.localtime:
                                  t -= time.timezone
                          if 'buffer' in self.plot_type:
                              if self.xmin is None:
                                  self.tmin = t
                              else:
                                  self.tmin = (
                                      self.getDateTime(t).replace(
-                                         hour=self.xmin,
-                                         minute=0,
+                                         hour=self.xmin,
+                                         minute=0,
                                          second=0) - self.getDateTime(0)).total_seconds()
                          self.data.setup()
                      if dataOut.useLocalTime and not self.localtime:
                          tm += time.timezone
-                     if self.xaxis is 'time' and self.data and (tm - self.tmin) >= self.xrange*60*60:
+                     if self.xaxis is 'time' and self.data and (tm - self.tmin) >= self.xrange*60*60:
                          self.save_counter = self.save_period
                          self.__plot()
                          self.xmin += self.xrange
                          self.__plot()
                      if self.data and self.pause:
                          figpause(10)

schainpy/model/io/jroIO_kamisr.py +205 -185

              from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
              from schainpy.model.data.jrodata import Voltage
-             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
              from numpy import imag
+             @MPDecorator
              class AMISRReader(ProcessingUnit):
                  '''
                  classdocs
                      '''
                      Constructor
                      '''
                      ProcessingUnit.__init__(self)
                      self.set = None
                      self.subset = None
                      self.extension_file = '.h5'
                      self.flagIsNewFile = 0
                      self.filename = ''
                      self.amisrFilePointer = None
-                     self.dataset = None
+                     #self.dataset = None
                      self.profileIndex = 0
                      self.beamCodeByFrame = None
                      self.radacTimeByFrame = None
                      self.dataset = None
                      self.__firstFile = True
                      self.buffer = None
                      self.timezone = 'ut'
                      self.__waitForNewFile = 20
-                     self.__filename_online = None
+                     self.__filename_online = None
                      #Is really necessary create the output object in the initializer
                      self.dataOut = Voltage()
+                     self.dataOut.error=False
                  def setup(self,path=None,
-                                 startDate=None,
-                                 endDate=None,
-                                 startTime=None,
+                                 startDate=None,
+                                 endDate=None,
+                                 startTime=None,
                                  endTime=None,
                                  walk=True,
                                  timezone='ut',
                                  nCode = 0,
                                  nBaud = 0,
                                  online=False):
+                     #print ("T",path)
                      self.timezone = timezone
                      self.all = all
                      self.online = online
                      self.code = code
                      self.nCode = int(nCode)
                      self.nBaud = int(nBaud)
                      #self.findFiles()
                      if not(online):
                          #Busqueda de archivos offline
                          self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
                      else:
                          self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
                      if not(self.filenameList):
                          print("There is no files into the folder: %s"%(path))
                          sys.exit(-1)
                      self.fileIndex = -1
-                     self.readNextFile(online)
+                     self.readNextFile(online)
                      '''
                      Add code
-                     '''
+                     '''
                      self.isConfig = True
                      pass
                  def readAMISRHeader(self,fp):
                      header = 'Raw11/Data/RadacHeader'
                      self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
                      self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
                      self.frequency =  fp.get('Rx/Frequency')
                      txAus = fp.get('Raw11/Data/Pulsewidth')
                      self.nblocks = self.pulseCount.shape[0] #nblocks
                      self.nprofiles = self.pulseCount.shape[1] #nprofile
                      self.nsa = self.nsamplesPulse[0,0] #ngates
                      self.nchannels = self.beamCode.shape[1]
                      self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
                      #self.__waitForNewFile = self.nblocks  # wait depending on the number of blocks since each block is 1 sec
                      self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
                      #filling radar controller header parameters
                      self.__ippKm = self.ippSeconds *.15*1e6 # in km
                      self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
                      self.__txB = 0
                      nWindows=1
-                     self.__nSamples = self.nsa
+                     self.__nSamples = self.nsa
                      self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
-                     self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
+                     self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
                      #for now until understand why the code saved is different (code included even though code not in tuf file)
                      #self.__codeType = 0
                     # self.__nCode = None
                      self.__nCode = self.nCode
                      self.__nBaud = self.nBaud
                      #self.__code = 0
                      #filling system header parameters
                      self.__nSamples = self.nsa
-                     self.newProfiles = self.nprofiles/self.nchannels
+                     self.newProfiles = self.nprofiles/self.nchannels
                      self.__channelList = list(range(self.nchannels))
                      self.__frequency = self.frequency[0][0]
                  def createBuffers(self):
-                     pass
+                     pass
                  def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
                      self.path = path
                      self.startDate = startDate
                      self.startTime = startTime
                      self.endTime = endTime
                      self.walk = walk
                  def __checkPath(self):
                      if os.path.exists(self.path):
                          self.status = 1
                      else:
                          self.status = 0
                          print('Path:%s does not exists'%self.path)
                      return
                  def __selDates(self, amisr_dirname_format):
                      try:
                          year = int(amisr_dirname_format[0:4])
                          month = int(amisr_dirname_format[4:6])
                          dom = int(amisr_dirname_format[6:8])
                          thisDate = datetime.date(year,month,dom)
                          if (thisDate>=self.startDate and thisDate <= self.endDate):
                              return amisr_dirname_format
                      except:
                          return None
                  def __findDataForDates(self,online=False):
                      if not(self.status):
                          return None
                      pat = '\d+.\d+'
                      dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
                      dirnameList = [x for x in dirnameList if x!=None]
                      else:
                          self.status = 0
                          return None
                  def __getTimeFromData(self):
                      startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
                      endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                          filename = self.filenameList[i]
                          fp = h5py.File(filename,'r')
                          time_str = fp.get('Time/RadacTimeString')
-                         startDateTimeStr_File = time_str[0][0].split('.')[0]
+                         startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
+                         #startDateTimeStr_File = "2019-12-16 09:21:11"
                          junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                          startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
-                         endDateTimeStr_File = time_str[-1][-1].split('.')[0]
+                         #endDateTimeStr_File = "2019-12-16 11:10:11"
+                         endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
                          junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                          endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                          fp.close()
+                         #print("check time", startDateTime_File)
                          if self.timezone == 'lt':
                              startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                              endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
                          if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
                              #self.filenameList.remove(filename)
                              filter_filenameList.append(filename)
                          if (endDateTime_File>=endDateTime_Reader):
                              break
                      filter_filenameList.sort()
                      self.filenameList = filter_filenameList
                      return 1
                  def __filterByGlob1(self, dirName):
                      filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
                      filter_files.sort()
                      filterDict.setdefault(dirName)
                      filterDict[dirName] = filter_files
                      return filterDict
                  def __getFilenameList(self, fileListInKeys, dirList):
                      for value in fileListInKeys:
                          dirName = list(value.keys())[0]
                          for file in value[dirName]:
                              filename = os.path.join(dirName, file)
                              self.filenameList.append(filename)
                  def __selectDataForTimes(self, online=False):
                      #aun no esta implementado el filtro for tiempo
                      if not(self.status):
                          return None
                      dirList = [os.path.join(self.path,x) for x in self.dirnameList]
                      fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
                      self.__getFilenameList(fileListInKeys, dirList)
                      if not(online):
                          #filtro por tiempo
                          else:
                              self.status = 0
                              return None
                      else:
                          #get the last file - 1
                          self.filenameList = [self.filenameList[-2]]
                      new_dirnameList = []
                      for dirname in self.dirnameList:
                          junk = numpy.array([dirname in x for x in self.filenameList])
                              new_dirnameList.append(dirname)
                      self.dirnameList = new_dirnameList
                      return 1
                  def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
                                          endTime=datetime.time(23,59,59),walk=True):
                      if endDate ==None:
                       startDate = datetime.datetime.utcnow().date()
                       endDate = datetime.datetime.utcnow().date()
                      self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
                      self.__checkPath()
                      self.__findDataForDates(online=True)
                      self.dirnameList = [self.dirnameList[-1]]
                      self.__selectDataForTimes(online=True)
                      return
                  def searchFilesOffLine(self,
                                          path,
                                          startDate,
                                          startTime=datetime.time(0,0,0),
                                          endTime=datetime.time(23,59,59),
                                          walk=True):
                      self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
                      self.__checkPath()
                      self.__findDataForDates()
                      self.__selectDataForTimes()
                      for i in range(len(self.filenameList)):
                          print("%s" %(self.filenameList[i]))
-                     return
+                     return
                  def __setNextFileOffline(self):
                      idFile = self.fileIndex
                          if not(idFile < len(self.filenameList)):
                              self.flagNoMoreFiles = 1
                              print("No more Files")
+                             self.dataOut.error = True
                              return 0
                          filename = self.filenameList[idFile]
                          amisrFilePointer = h5py.File(filename,'r')
                          break
                      self.flagIsNewFile = 1
                      print("Setting the file: %s"%self.filename)
                      return 1
                  def __setNextFileOnline(self):
                      filename = self.filenameList[0]
                      if self.__filename_online != None:
                              self.__selectDataForTimes(online=True)
                              filename = self.filenameList[0]
                              wait += 1
                      self.__filename_online = filename
                      self.amisrFilePointer = h5py.File(filename,'r')
                      self.flagIsNewFile = 1
                      self.filename = filename
                      print("Setting the file: %s"%self.filename)
                      return 1
                  def readData(self):
                      buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
                      re = buffer[:,:,:,0]
                      im = buffer[:,:,:,1]
                      dataset = re + im*1j
                      self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
                      timeset = self.radacTime[:,0]
                      return dataset,timeset
                  def reshapeData(self):
-                 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
+                 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
                      channels = self.beamCodeByPulse[0,:]
                      nchan = self.nchannels
                      #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
                      nblocks = self.nblocks
                      nsamples = self.nsa
                      #Dimensions : nChannels, nProfiles, nSamples
-                     new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
+                     new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
                      ############################################
                      for thisChannel in range(nchan):
                          new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
                      new_block = numpy.transpose(new_block, (1,0,2,3))
                      new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
-                     return new_block
+                     return new_block
                  def updateIndexes(self):
                      pass
                  def fillJROHeader(self):
                      #fill radar controller header
-                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
+                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
                                                                                    txA=self.__txA,
                                                                                    txB=0,
                                                                                    nWindows=1,
                                                                                    nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                    code = self.__code,
                                                                                    fClock=1)
                      #fill system header
                      self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                  nProfiles=self.newProfiles,
                                                                  nChannels=len(self.__channelList),
                                                                  adcResolution=14,
-                                                                 pciDioBusWith=32)
+                                                                 pciDioBusWidth=32)
                      self.dataOut.type = "Voltage"
                      self.dataOut.data = None
                      self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
              #        self.dataOut.nChannels = 0
              #        self.dataOut.nHeights = 0
                      self.dataOut.nProfiles = self.newProfiles*self.nblocks
                      #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                      ranges = numpy.reshape(self.rangeFromFile.value,(-1))
                      self.dataOut.heightList =  ranges/1000.0 #km
                      self.dataOut.channelList = self.__channelList
                      self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
              #        self.dataOut.channelIndexList = None
                      self.dataOut.flagNoData = True
-                     #Set to TRUE if the data is discontinuous
+                     #Set to TRUE if the data is discontinuous
                      self.dataOut.flagDiscontinuousBlock = False
                      self.dataOut.utctime = None
                      #self.dataOut.timeZone = -5 #self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                      if self.timezone == 'lt':
                          self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
-                     else:
+                     else:
                          self.dataOut.timeZone = 0 #by default time is UTC
                      self.dataOut.dstFlag = 0
                      self.dataOut.errorCount = 0
                      self.dataOut.nCohInt = 1
                      self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                      self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                      self.dataOut.flagShiftFFT = False
                      self.dataOut.ippSeconds = self.ippSeconds
-                     #Time interval between profiles
+                     #Time interval between profiles
                      #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                      self.dataOut.frequency = self.__frequency
                      self.dataOut.realtime = self.online
                      pass
                  def readNextFile(self,online=False):
                      if not(online):
                          newFile = self.__setNextFileOffline()
                      else:
-                         newFile = self.__setNextFileOnline()
+                         newFile = self.__setNextFileOnline()
                      if not(newFile):
                          return 0
                      #if self.__firstFile:
                      self.readAMISRHeader(self.amisrFilePointer)
                      self.createBuffers()
                      self.fillJROHeader()
                      #self.__firstFile = False
                      self.dataset,self.timeset = self.readData()
                      if self.endDate!=None:
                       endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
                       time_str = self.amisrFilePointer.get('Time/RadacTimeString')
-                      startDateTimeStr_File = time_str[0][0].split('.')[0]
+                      startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
                       junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
                       startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
                       if self.timezone == 'lt':
                        startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
                       if (startDateTime_File>endDateTime_Reader):
                           return 0
                      self.jrodataset = self.reshapeData()
                      #----self.updateIndexes()
                      self.profileIndex = 0
                      return 1
                  def __hasNotDataInBuffer(self):
                      if self.profileIndex >= (self.newProfiles*self.nblocks):
                          return 1
                      return 0
                  def getData(self):
                      if self.flagNoMoreFiles:
                          self.dataOut.flagNoData = True
                          return 0
                      if self.__hasNotDataInBuffer():
                          if not (self.readNextFile(self.online)):
                              return 0
-                     if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
-                         self.dataOut.flagNoData = True
+                     if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
+                         self.dataOut.flagNoData = True
                          return 0
                      #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
                      self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
+                     #print("R_t",self.timeset)
                      #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
                      #verificar basic header de jro data y ver si es compatible con este valor
                      #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
                      indexprof = numpy.mod(self.profileIndex, self.newProfiles)
                      indexblock = self.profileIndex/self.newProfiles
-                     #print indexblock, indexprof
-                     self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
+                     #print (indexblock, indexprof)
+                     diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
+                     diffUTC = 0
+                     t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
+                     #cambio posible 18/02/2020
+                     #print("utc :",indexblock," __ ",t_comp)
+                     #print(numpy.shape(self.timeset))
+                     self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
+                     #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
+                     #print(self.dataOut.utctime)
                      self.dataOut.profileIndex = self.profileIndex
                      self.dataOut.flagNoData = False
              #         if indexprof == 0:
              #             print self.dataOut.utctime
                      self.profileIndex += 1
                      return self.dataOut.data
                  def run(self, **kwargs):
                      '''
                      This method will be called many times so here you should put all your code
                      '''
                      if not self.isConfig:
                          self.setup(**kwargs)
                          self.isConfig = True
                      self.getData()

schainpy/model/io/jroIO_param.py +36 -36

                      except IOError:
                          traceback.print_exc()
                          raise IOError("The file %s can't be opened" %(filename))
                      #In case has utctime attribute
                      grp2 = grp1['utctime']
               #         thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
                  setType = None
                  def __init__(self):
                      Operation.__init__(self)
                      return
                          dsDict['variable'] = self.dataList[i]
                          #---------------------    Conditionals    ------------------------
                          #There is no data
                          if dataAux is None:
                              return 0
                          if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
                          return False
                  def setNextFile(self):
                      ext = self.ext
                      path = self.path
                      setFile = self.setFile
                      if os.path.exists(fullpath):
                          filesList = os.listdir( fullpath )
-                         filesList = [k for k in filesList if 'M' in k]
+                         ##filesList = [k for k in filesList if 'M' in k]
                          if len( filesList ) > 0:
                              filesList = sorted( filesList, key=str.lower )
                              filen = filesList[-1]
                              for j in range(dsInfo['dsNumber']):
                                  dsInfo = dsList[i]
                                  tableName = dsInfo['dsName']
                                  if dsInfo['nDim'] == 3:
                                      shape = dsInfo['shape'].astype(int)
                      self.dataOut = dataOut
                      if not(self.isConfig):
-                         self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
+                         self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                                     metadataList=metadataList, dataList=dataList, mode=mode,
                                     setType=setType)
                      self.putData()
                      return
              @MPDecorator
              class ParameterReader(Reader, ProcessingUnit):
                      self.set_kwargs(**kwargs)
                      if not self.ext.startswith('.'):
-                         self.ext = '.{}'.format(self.ext)
+                         self.ext = '.{}'.format(self.ext)
                      if self.online:
                          log.log("Searching files in online mode...", self.name)
                          for nTries in range(self.nTries):
                              fullpath = self.searchFilesOnLine(self.path, self.startDate,
-                                 self.endDate, self.expLabel, self.ext, self.walk,
+                                 self.endDate, self.expLabel, self.ext, self.walk,
                                  self.filefmt, self.folderfmt)
                              try:
                                  fullpath = next(fullpath)
                              except:
                                  fullpath = None
                              if fullpath:
                                  break
                              log.warning(
                                  'Waiting {} sec for a valid file in {}: try {} ...'.format(
-                                     self.delay, self.path, nTries + 1),
+                                     self.delay, self.path, nTries + 1),
                                  self.name)
                              time.sleep(self.delay)
                          if not(fullpath):
                              raise schainpy.admin.SchainError(
-                                 'There isn\'t any valid file in {}'.format(self.path))
+                                 'There isn\'t any valid file in {}'.format(self.path))
                          pathname, filename = os.path.split(fullpath)
                          self.year = int(filename[1:5])
                          self.doy = int(filename[5:8])
-                         self.set = int(filename[8:11]) - 1
+                         self.set = int(filename[8:11]) - 1
                      else:
                          log.log("Searching files in {}".format(self.path), self.name)
-                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
+                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                              self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                      self.setNextFile()
                      return
                  def readFirstHeader(self):
                      '''Read metadata and data'''
-                     self.__readMetadata()
+                     self.__readMetadata()
                      self.__readData()
                      self.__setBlockList()
                      self.blockIndex = 0
                      return
                  def __setBlockList(self):
                              else:
                                  data = gp[name].value
                                  listMetaname.append(name)
-                                 listMetadata.append(data)
+                                 listMetadata.append(data)
                      elif self.metadata:
                          metadata = json.loads(self.metadata)
                          listShapes = {}
                      self.listShapes = listShapes
                      self.listMetaname = listMetaname
-                     self.listMeta = listMetadata
+                     self.listMeta = listMetadata
                      return
                      listdataname = []
                      listdata = []
                      if 'Data' in self.fp:
                          grp = self.fp['Data']
                          for item in list(grp.items()):
                                  for i in range(dim):
                                      array.append(grp[name]['table{:02d}'.format(i)].value)
                                  array = numpy.array(array)
                              listdata.append(array)
                      elif self.metadata:
                          metadata = json.loads(self.metadata)
                      self.listDataname = listdataname
                      self.listData = listdata
                      return
                  def getData(self):
                      for i in range(len(self.listMeta)):
                  lastTime = None
                  def __init__(self):
                      Operation.__init__(self)
                      return
                              dsDict['nDim'] = len(dataAux.shape)
                              dsDict['shape'] = dataAux.shape
                              dsDict['dsNumber'] = dataAux.shape[0]
                          dsList.append(dsDict)
                          tableList.append((self.dataList[i], dsDict['nDim']))
                          self.lastTime = currentTime
                          self.currentDay = dataDay
                          return False
                      timeDiff = currentTime - self.lastTime
                      #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
                      self.dataOut = dataOut
                      if not(self.isConfig):
-                         self.setup(path=path, blocksPerFile=blocksPerFile,
+                         self.setup(path=path, blocksPerFile=blocksPerFile,
                                     metadataList=metadataList, dataList=dataList,
                                     setType=setType)
                      self.putData()
                      return
                  def setNextFile(self):
                      ext = self.ext
                      path = self.path
                      setFile = self.setFile
                      return
                  def writeData(self, fp):
                      grp = fp.create_group("Data")
                      dtsets = []
                      data = []
                      for dsInfo in self.dsList:
                          if dsInfo['nDim'] == 0:
                              ds = grp.create_dataset(
-                                 dsInfo['variable'],
+                                 dsInfo['variable'],
                                  (self.blocksPerFile, ),
-                                 chunks=True,
+                                 chunks=True,
                                  dtype=numpy.float64)
                              dtsets.append(ds)
                              data.append((dsInfo['variable'], -1))
                              sgrp = grp.create_group(dsInfo['variable'])
                              for i in range(dsInfo['dsNumber']):
                                  ds = sgrp.create_dataset(
-                                     'table{:02d}'.format(i),
+                                     'table{:02d}'.format(i),
                                      (self.blocksPerFile, ) + dsInfo['shape'][1:],
                                      chunks=True)
                                  dtsets.append(ds)
                      fp.flush()
                      log.log('Creating file: {}'.format(fp.filename), self.name)
                      self.ds = dtsets
                      self.data = data
                      self.firsttime = True

schainpy/model/proc/jroproc_base.py +37 -40

              Jan 2018
              Abstract:
                  Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
-                 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
-                 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
+                 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
+                 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
              Based on:
                  $Author: murco $
                  """
                  Update - Jan 2018 - MULTIPROCESSING
-                 All the "call" methods present in the previous base were removed.
+                 All the "call" methods present in the previous base were removed.
                  The majority of operations are independant processes, thus
-                 the decorator is in charge of communicate the operation processes
+                 the decorator is in charge of communicate the operation processes
                  with the proccessing unit via IPC.
                  The constructor does not receive any argument. The remaining methods
                  are related with the operations to execute.
                  """
                  proc_type = 'processing'
                  def addOperation(self, conf, operation):
                      """
-                     This method is used in the controller, and update the dictionary containing the operations to execute. The dict
+                     This method is used in the controller, and update the dictionary containing the operations to execute. The dict
                      posses the id of the operation process (IPC purposes)
                          Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
                      self.operations.append(
                          (operation, conf.type, conf.id, conf.getKwargs()))
                      if 'plot' in self.name.lower():
                          self.plots.append(operation.CODE)
                      return
              class InputQueue(Thread):
                  '''
                  Class to hold input data for Proccessing Units and external Operations,
                  '''
                  def get(self):
                      if not self.islocked and self.size/1000000 > 512:
-                         self.lock.n.value += 1
+                         self.lock.n.value += 1
                          self.islocked = True
                          self.lock.clear()
                      elif self.islocked and self.size/1000000 <= 512:
                          self.islocked = False
                          self.lock.n.value -= 1
                          if self.lock.n.value == 0:
-                             self.lock.set()
+                             self.lock.set()
                      obj = self.queue.get()
                      self.size -= sys.getsizeof(obj)
                      return pickle.loads(obj)
              def MPDecorator(BaseClass):
                  """
                  Multiprocessing class decorator
                  This function add multiprocessing features to a BaseClass. Also, it handle
-                 the communication beetween processes (readers, procUnits and operations).
+                 the communication beetween processes (readers, procUnits and operations).
                  """
                  class MPClass(BaseClass, Process):
                          self.t = time.time()
                          self.name = BaseClass.__name__
                          self.__doc__ = BaseClass.__doc__
                          if 'plot' in self.name.lower() and not self.name.endswith('_'):
                              self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
-                         self.start_time = time.time()
+                         self.start_time = time.time()
                          self.id = args[0]
                          self.inputId = args[1]
                          self.project_id = args[2]
                          '''
                          self.queue.start()
                      def listen(self):
                          '''
                          This function waits for objects
                          '''
-                         return self.queue.get()
+                         return self.queue.get()
                      def set_publisher(self):
                          '''
-                         This function create a zmq socket for publishing objects.
+                         This function create a zmq socket for publishing objects.
                          '''
                          time.sleep(0.5)
                          c = zmq.Context()
                          self.sender = c.socket(zmq.PUB)
                          self.sender.connect(
                          '''
                          This function publish an object, to an specific topic.
                          It blocks publishing when receiver queue is full to avoid data loss
-                         '''
+                         '''
                          if self.inputId is None:
                              self.lock.wait()
                          self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
                      def runReader(self):
                          '''
                          Run fuction for read units
                              try:
                                  BaseClass.run(self, **self.kwargs)
                              except:
-                                 err = traceback.format_exc()
+                                 err = traceback.format_exc()
                                  if 'No more files' in err:
                                      log.warning('No more files to read', self.name)
                                  else:
                                      self.err_queue.put('{}|{}'.format(self.name, err))
-                                 self.dataOut.error = True
+                                 self.dataOut.error = True
                              for op, optype, opId, kwargs in self.operations:
                                  if optype == 'self' and not self.dataOut.flagNoData:
                                      op(**kwargs)
                                  continue
                              self.publish(self.dataOut, self.id)
-                             if self.dataOut.error:
+                             if self.dataOut.error:
                                  break
                          time.sleep(0.5)
                          '''
                          while True:
-                             self.dataIn = self.listen()
+                             self.dataIn = self.listen()
                              if self.dataIn.flagNoData and self.dataIn.error is None:
                                  continue
                              elif self.dataIn.error:
                                  self.dataOut.error = self.dataIn.error
                                  self.dataOut.flagNoData = True
                              for op, optype, opId, kwargs in self.operations:
                                  if optype == 'self' and not self.dataOut.flagNoData:
                                      op(**kwargs)
                                  elif optype == 'other' and not self.dataOut.flagNoData:
                                      self.dataOut = op.run(self.dataOut, **kwargs)
-                                 elif optype == 'external' and not self.dataOut.flagNoData:
+                                 elif optype == 'external' and not self.dataOut.flagNoData:
                                      self.publish(self.dataOut, opId)
                              self.publish(self.dataOut, self.id)
                              for op, optype, opId, kwargs in self.operations:
-                                 if optype == 'external' and self.dataOut.error:
+                                 if optype == 'external' and self.dataOut.error:
                                      self.publish(self.dataOut, opId)
                              if self.dataOut.error:
                                  break
                          time.sleep(0.5)
                      def runOp(self):
                          Run function for external operations (this operations just receive data
                          ex: plots, writers, publishers)
                          '''
                          while True:
                              dataOut = self.listen()
                                      self.err_queue.put('{}|{}'.format(self.name, traceback.format_exc()))
                                      dataOut.error = True
                              else:
-                                 break
+                                 break
                      def run(self):
                          if self.typeProc is "ProcUnit":
                              if self.inputId is not None:
                                  self.subscribe()
                              self.set_publisher()
                              if 'Reader' not in BaseClass.__name__:
                                  self.runProc()
                              else:
                                  self.runReader()
                          elif self.typeProc is "Operation":
                              self.subscribe()

schainpy/model/proc/jroproc_parameters.py 0 0

	1		NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff

schainpy/model/proc/jroproc_spectra.py +31 -31

                      # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
                      self.dataOut.channelList = range(len(channelIndexList))
                      self.__selectPairsByChannel(channelIndexList)
                      return 1
                  def selectFFTs(self, minFFT, maxFFT ):
                      """
-                     Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
+                     Selecciona un bloque de datos en base a un grupo de valores de puntos FFTs segun el rango
                      minFFT<= FFT <= maxFFT
                      """
                      if (minFFT > maxFFT):
                          raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minFFT, maxFFT))
                      self.selectFFTsByIndex(minIndex, maxIndex)
                      return 1
                  def setH0(self, h0, deltaHeight = None):
                      if not deltaHeight:
                          deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                      nHeights = self.dataOut.nHeights
                      newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
                      self.dataOut.heightList = newHeiRange
                  def selectHeights(self, minHei, maxHei):
                      """
                      Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
 si el metodo se ejecuto con exito caso contrario devuelve 0
                      """
                      if (minHei > maxHei):
                          raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (minHei, maxHei))
                          maxIndex = len(heights)
                      self.selectHeightsByIndex(minIndex, maxIndex)
                      return 1
                  def selectFFTsByIndex(self, minIndex, maxIndex):
                      """
                      """
                      if (minIndex < 0) or (minIndex > maxIndex):
                      self.dataOut.data_spc = data_spc
                      self.dataOut.data_cspc = data_cspc
                      self.dataOut.data_dc = data_dc
                      self.dataOut.ippSeconds = self.dataOut.ippSeconds*(self.dataOut.nFFTPoints / numpy.shape(data_cspc)[1])
                      self.dataOut.nFFTPoints = numpy.shape(data_cspc)[1]
                      self.dataOut.profilesPerBlock = numpy.shape(data_cspc)[1]
                          xx_inv = numpy.linalg.inv(xx)
                          xx_aux = xx_inv[0, :]
-                         for ich in range(num_chan):
+                         for ich in range(num_chan):
                              yy = jspectra[ich, ind_vel, :]
                              jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
                      return 1
                  def removeInterference2(self):
                      cspc = self.dataOut.data_cspc
                      spc = self.dataOut.data_spc
-                     Heights = numpy.arange(cspc.shape[2])
+                     Heights = numpy.arange(cspc.shape[2])
                      realCspc = numpy.abs(cspc)
                      for i in range(cspc.shape[0]):
                          LinePower= numpy.sum(realCspc[i], axis=0)
                          Threshold = numpy.amax(LinePower)-numpy.sort(LinePower)[len(Heights)-int(len(Heights)*0.1)]
                          InterferenceSum = numpy.sum( realCspc[i,:,SelectedHeights], axis=0 )
                          InterferenceThresholdMin = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.98)]
                          InterferenceThresholdMax = numpy.sort(InterferenceSum)[int(len(InterferenceSum)*0.99)]
                          InterferenceRange = numpy.where( ([InterferenceSum > InterferenceThresholdMin]))# , InterferenceSum < InterferenceThresholdMax]) )
                          #InterferenceRange = numpy.where( ([InterferenceRange < InterferenceThresholdMax]))
                          if len(InterferenceRange)<int(cspc.shape[1]*0.3):
                              cspc[i,InterferenceRange,:] = numpy.NaN
                      self.dataOut.data_cspc = cspc
                  def removeInterference(self,  interf = 2,hei_interf = None, nhei_interf = None, offhei_interf = None):
                      jspectra = self.dataOut.data_spc
                      if n is not None:
                          self.n = int(n)
                      else:
                          self.__integrationtime = int(timeInterval)
                          self.n = None
                          self.__byTime = True
                  def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
                      if n == 1:
                          return
                      dataOut.flagNoData = True
                      if not self.isConfig:
                          dataOut.data_spc = avgdata_spc
                          dataOut.data_cspc = avgdata_cspc
-                         dataOut.data_dc = avgdata_dc
+                         dataOut.data_dc = avgdata_dc
                          dataOut.nIncohInt *= self.n
                          dataOut.utctime = avgdatatime
                          dataOut.flagNoData = False
-                     return dataOut
  No newline at end of file
+                     return dataOut

schainpy/model/proc/jroproc_voltage.py +20 -21

              @MPDecorator
-             class VoltageProc(ProcessingUnit):
+             class VoltageProc(ProcessingUnit):
                  def __init__(self):
                      ProcessingUnit.__init__(self)
                      self.dataOut.data = data
                      # self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
                      self.dataOut.channelList = range(len(channelIndexList))
                      return 1
                  def selectHeights(self, minHei=None, maxHei=None):
                          """
                          Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
                          """
-                         buffer = self.dataOut.data[:, :, 0:int(self.dataOut.nHeights-r)]
+                         buffer = self.dataOut.data[:, :, 0:int(self.dataOut.nHeights-r)]
                          buffer = buffer.reshape(self.dataOut.nChannels, self.dataOut.nProfiles, int(self.dataOut.nHeights/window), window)
                          buffer = numpy.sum(buffer,3)
                          # print self.__bufferStride[self.__profIndexStride - 1]
                          # raise
                          return self.__bufferStride[self.__profIndexStride - 1]
                      return None, None
                  def integrate(self, data, datatime=None):
                      avgdatatime = self.__initime
                      deltatime = datatime - self.__lastdatatime
                      if not self.__withOverlapping:
                          self.__initime = datatime
                      else:
                      avgdatatime = (times - 1) * timeInterval + dataOut.utctime
                      self.__dataReady = True
                      return avgdata, avgdatatime
                  def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
                      if not self.isConfig:
                          avgdata, avgdatatime = self.integrateByBlock(dataOut)
                          dataOut.nProfiles /= self.n
                      else:
-                         if stride is None:
+                         if stride is None:
                              avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
                          else:
                              avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
                      #   dataOut.timeInterval *= n
                      dataOut.flagNoData = True
                      self.nCode = len(code)
                      self.nBaud = len(code[0])
                      if (osamp != None) and (osamp >1):
                          self.osamp = osamp
                          self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
                      #Frequency
                      __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
                      __codeBuffer[:,0:self.nBaud] = self.code
                      self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
                      junk = junk.flatten()
                      code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
                      profilesList = range(self.__nProfiles)
-                     for i in range(self.__nChannels):
-                         for j in profilesList:
-                             self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
-                     return self.datadecTime
+                     for i in range(self.__nChannels):
+                         for j in profilesList:
+                             self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
+                     return self.datadecTime
                  def __convolutionByBlockInFreq(self, data):
                      return data
                  def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
                      if dataOut.flagDecodeData:
                      self.__nProfiles = dataOut.nProfiles
                      datadec = None
                      if mode == 3:
                          mode = 0
                          if shape[2] % n != 0:
                              raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
                          new_shape = shape[0], shape[1]*n, int(shape[2]/n)
                          dataOut.data = numpy.reshape(dataOut.data, new_shape)
                          dataOut.flagNoData = False

schainpy/model/utils/jroutils_ftp.py +27 -27

                      self.mutex.acquire()
                      #         init = time.time()
+                     #
+                     #
                      #         while(self.bussy):
                      #             sleep(0.1)
                      #             if time.time() - init > 2*self.period:
                      #                 return 0
                      self.fileList = fileList
                      self.mutex.release()
                      return 1
                          if self.stopFlag:
                              break
                          #   self.bussy = True
                          self.mutex.acquire()
                      """
                          This method is used to set SSH parameters and establish a connection to a remote server
                          Inputs:
-                             server    - remote server IP Address
-                             username    - remote server Username
+                             server    - remote server IP Address
+                             username    - remote server Username
                              password    - remote server password
                              remotefolder    - remote server current working directory
                          Return: void
-                         Affects:
+                         Affects:
                              self.status        - in case of error or fail connection this parameter is set to 0 else 1
                      """
                  def __execute(self, command):
                      """
                          __execute a command on remote server
                          Input:
                              command    - Exmaple 'ls -l'
                          Return:
 in error case else 1
                      """
                  def mkdir(self, remotefolder):
                      """
                          mkdir is used to make a new directory in remote server
                          Input:
                              remotefolder    - directory name
                          Return:
 in error case else 1
                      """
                  def cd(self, remotefolder):
                      """
                          cd is used to change remote working directory on server
                          Input:
                              remotefolder    - current working directory
                          Affects:
                              self.remotefolder
-                         Return:
+                         Return:
 in case of error else 1
                      """
                      if not self.status:
                      ProcessingUnit.__init__(self, **kwargs)
                      self.isConfig = False
-                     self.clientObj = None
+                     self.clientObj = None
                  def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
                      self.clientObj = None
                          self.init = time.time()
                          self.setup(**kwargs)
                          self.isConfig = True
                      if not self.clientObj.is_alive():
                          print("[Remote Server]: Restarting connection ")
                          self.setup(**kwargs)
                      if time.time() - self.init >= self.period:
                          fullfilenameList = self.findFiles()
                      try:
                          self.ftp = ftplib.FTP(self.server)
                          self.ftp.login(self.username,self.password)
-                         self.ftp.cwd(self.remotefolder)
+                         self.ftp.cwd(self.remotefolder)
                          #   print 'Connect to FTP Server: Successfully'
                      except ftplib.all_errors:
                          print('Error FTP Service')
                          self.status = 1
                          self.counter = 0
-                     self.status = 1
  No newline at end of file
+                     self.status = 1

schainpy/model/utils/jroutils_publish.py +20 -20

              def get_plot_code(s):
                  label = s.split('_')[0]
                  codes = [key for key in PLOT_CODES if key in label]
-                 if codes:
+                 if codes:
                      return PLOT_CODES[codes[0]]
                  else:
                      return 24
                      self.counter = 0
                      self.delay = kwargs.get('delay', 0)
                      self.cnt = 0
-                     self.verbose = verbose
+                     self.verbose = verbose
                      context = zmq.Context()
                      self.zmq_socket = context.socket(zmq.PUSH)
                      server = kwargs.get('server', 'zmq.pipe')
                  def publish_data(self):
                      self.dataOut.finished = False
                      if self.verbose:
                          log.log(
                              'Sending {} - {}'.format(self.dataOut.type, self.dataOut.datatime),
                      time.sleep(self.delay)
                  def close(self):
                      self.dataOut.finished = True
                      self.zmq_socket.send_pyobj(self.dataOut)
                      time.sleep(0.1)
                      self.zmq_socket.close()
              class ReceiverData(ProcessingUnit):
                              self.ftp.close()
                          self.ftp = None
                          self.ready = False
-                         return
+                         return
                      try:
                          self.ftp.login(self.username, self.password)
                  def upload(self, src, dst):
                      log.log('Uploading {} -> {} '.format(
-                         src.split('/')[-1], dst.split('/')[-1]),
-                         self.name,
+                         src.split('/')[-1], dst.split('/')[-1]),
+                         self.name,
                          nl=False
                          )
                      fp.close()
                      log.success('OK', tag='')
                      return 1
                  def send_files(self):
                      for x, pattern in enumerate(self.patterns):
                              srcname = self.find_files(local, ext)
                              src = os.path.join(local, srcname)
                              if os.path.getmtime(src) < time.time() - 30*60:
-                                 log.warning('Skipping old file {}'.format(srcname))
+                                 log.warning('Skipping old file {}'.format(srcname))
                                  continue
                              if srcname is None or srcname == self.latest[x]:
-                                 log.warning('File alreday uploaded {}'.format(srcname))
+                                 log.warning('File alreday uploaded {}'.format(srcname))
                                  continue
                              if 'png' in ext:
                                  dstname = self.getftpname(srcname, int(exp_code), int(sub_exp_code))
                              else:
-                                 dstname = srcname
+                                 dstname = srcname
                              dst = os.path.join(remote, dstname)
                              if self.upload(src, dst):
                                  self.times[x] = time.time()
                                  self.latest[x] = srcname
-                             else:
+                             else:
                                  self.ready = False
-                                 break
+                                 break
                  def run(self, dataOut, server, username, password, timeout=10, **kwargs):
                      if not self.isConfig:
                          self.setup(
-                             server=server,
-                             username=username,
-                             password=password,
-                             timeout=timeout,
+                             server=server,
+                             username=username,
+                             password=password,
+                             timeout=timeout,
                              **kwargs
                              )
                          self.isConfig = True

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages