schain-jc Commit - r1112:4bbef2dc4ffc

set new matplotlib version in setup, improved searchFiles for multiprocessing

Juan C. Espinoza -

r1112:4bbef2dc4ffc

parent child

Context file:

r1112:4bbef2dc4ffc

Collapse all files

schainpy/controller.py +10 -4

                  op = rconf.getOperationObj('run')
                  dt1 = op.getParameterValue('startDate')
                  dt2 = op.getParameterValue('endDate')
+                 tm1 = op.getParameterValue('startTime')
+                 tm2 = op.getParameterValue('endTime')
                  days = (dt2 - dt1).days
                  for day in range(days + 1):
                      paths, files = reader.searchFilesOffLine(path=rconf.path,
                                                               startDate=dt,
                                                               endDate=dt,
+                                                              startTime=tm1,
+                                                              endTime=tm2,
                                                               ext=DTYPES[rconf.datatype])
                      nFiles = len(files)
                      if nFiles == 0:
                          continue
-                     skip = int(math.ceil(nFiles / n))
+                     skip = int(math.ceil(nFiles / n))
                      while nFiles > cursor * skip:
                          rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
                                       skip=skip)
                      print '*' * 60
                      print
                      self.id = str(id)
-                     self.description = description
+                     self.description = description
                  def update(self, name, description):
                  def run(self):
                      log.success('Starting {}'.format(self.name))
+                     self.start_time = time.time()
                      self.createObjects()
                      self.connectObjects()
                          procUnitConfObj = self.procUnitConfObjDict[procKey]
                          procUnitConfObj.close()
-                     log.success('{} finished'.format(self.name))
+                     log.success('{} finished (time: {}s)'.format(
+                         self.name,
+                         time.time()-self.start_time))

schainpy/model/data/jroheaderIO.py +2 -2

                          nBaud.tofile(fp)
                          code1 = (self.code + 1.0)/2.
-                         for ic in range(self.nCode):
-                             tempx = numpy.zeros(numpy.ceil(self.nBaud/32.))
+                         for ic in range(self.nCode):
+                             tempx = numpy.zeros(int(numpy.ceil(self.nBaud/32.)))
                              start = 0
                              end = 32
                              for i in range(len(tempx)):

schainpy/model/io/jroIO_base.py +16 -25

              from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
              from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
+             from schainpy.utils import log
              LOCALTIME = True
              def isRadarFile(file):
-                 try:
+                 try:
                      year = int(file[1:5])
                      doy = int(file[5:8])
                      set = int(file[8:11])
              def getDateFromRadarFile(file):
-                 try:
+                 try:
                      year = int(file[1:5])
                      doy = int(file[5:8])
-                     set = int(file[8:11])
+                     set = int(file[8:11])
                  except:
                      return None
                      filenameList = []
                      datetimeList = []
                      for thisPath in pathList:
                          fileList = glob.glob1(thisPath, "*%s" % ext)
                          fileList.sort()
-                         skippedFileList = []
-                         if cursor is not None and skip is not None:
-                             if skip == 0:
-                                 skippedFileList = []
-                             else:
-                                 skippedFileList = fileList[cursor *
-                                                            skip: cursor * skip + skip]
-                         else:
-                             skippedFileList = fileList
-                         for file in skippedFileList:
+                         for file in fileList:
                              filename = os.path.join(thisPath, file)
                              filenameList.append(filename)
                              datetimeList.append(thisDatetime)
+                     if cursor is not None and skip is not None:
+                         filenameList = filenameList[cursor * skip:cursor * skip + skip]
+                         datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
                      if not(filenameList):
                          print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
                          return [], []
-                     print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
-                     print
+                     print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
                      # for i in range(len(filenameList)):
                      #     print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                      if self.dataOut.datatime.date() > self.fileDate:
                          setFile = 0
                          self.nTotalBlocks = 0
-                     filen = '%s%4.4d%3.3d%3.3d%s' % (
-                         self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
+                     filen = '{}{:04d}{:03d}{:03d}{}'.format(
+                             self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
                      filename = os.path.join(path, subfolder, filen)
                      self.ext = ext.lower()
                      self.path = path
                      if set is None:
                          self.setFile = -1
                      else:
-                         self.setFile = set - 1
+                         self.setFile = set - 1
                      self.blocksPerFile = blocksPerFile

setup.py +1 -1

                    install_requires=[
                        "scipy >= 0.14.0",
                        "h5py >= 2.2.1",
-                       "matplotlib >= 1.4.2",
+                       "matplotlib >= 2.0.0",
                        "pyfits >= 3.4",
                        "paramiko >= 2.1.2",
                        "paho-mqtt >= 1.2",

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages