schain-jc Commit - r1112:4bbef2dc4ffc

set new matplotlib version in setup, improved searchFiles for multiprocessing

Juan C. Espinoza -

r1112:4bbef2dc4ffc

parent child

Context file:

r1112:4bbef2dc4ffc

Collapse all files

schainpy/controller.py +10 -4

                 op = rconf.getOperationObj('run')
                 dt1 = op.getParameterValue('startDate')
                 dt2 = op.getParameterValue('endDate')
+                tm1 = op.getParameterValue('startTime')
+                tm2 = op.getParameterValue('endTime')
                 days = (dt2 - dt1).days
                 for day in range(days + 1):
                     paths, files = reader.searchFilesOffLine(path=rconf.path,
                                                              startDate=dt,
                                                              endDate=dt,
+                                                             startTime=tm1,
+                                                             endTime=tm2,
                                                              ext=DTYPES[rconf.datatype])
                     nFiles = len(files)
                     if nFiles == 0:
                         continue
                     skip = int(math.ceil(nFiles / n))
                     while nFiles > cursor * skip:
                         rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
                                      skip=skip)
                     print '*' * 60
                     print
                     self.id = str(id)
                     self.description = description
                 def update(self, name, description):
                 def run(self):
                     log.success('Starting {}'.format(self.name))
+                    self.start_time = time.time()
                     self.createObjects()
                     self.connectObjects()
                         procUnitConfObj = self.procUnitConfObjDict[procKey]
                         procUnitConfObj.close()
-                    log.success('{} finished'.format(self.name))
+                    log.success('{} finished (time: {}s)'.format(
+                        self.name,
+                        time.time()-self.start_time))

schainpy/model/data/jroheaderIO.py +2 -2

                         nBaud.tofile(fp)
                         code1 = (self.code + 1.0)/2.
                         for ic in range(self.nCode):
-                            tempx = numpy.zeros(numpy.ceil(self.nBaud/32.))
+                            tempx = numpy.zeros(int(numpy.ceil(self.nBaud/32.)))
                             start = 0
                             end = 32
                             for i in range(len(tempx)):

schainpy/model/io/jroIO_base.py +16 -25

             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
+            from schainpy.utils import log
             LOCALTIME = True
             def isRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
             def getDateFromRadarFile(file):
                 try:
                     year = int(file[1:5])
                     doy = int(file[5:8])
                     set = int(file[8:11])
                 except:
                     return None
                     filenameList = []
                     datetimeList = []
                     for thisPath in pathList:
                         fileList = glob.glob1(thisPath, "*%s" % ext)
                         fileList.sort()
-                        skippedFileList = []
+                        for file in fileList:
-                        if cursor is not None and skip is not None:
-                            if skip == 0:
-                                skippedFileList = []
-                            else:
-                                skippedFileList = fileList[cursor *
-                                                           skip: cursor * skip + skip]
-                        else:
-                            skippedFileList = fileList
-                        for file in skippedFileList:
                             filename = os.path.join(thisPath, file)
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
+                    if cursor is not None and skip is not None:
+                        filenameList = filenameList[cursor * skip:cursor * skip + skip]
+                        datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
                     if not(filenameList):
                         print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path)
                         return [], []
                     print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime)
-                    print
                     # for i in range(len(filenameList)):
                     #     print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
                     if self.dataOut.datatime.date() > self.fileDate:
                         setFile = 0
                         self.nTotalBlocks = 0
-                    filen = '%s%4.4d%3.3d%3.3d%s' % (
+                    filen = '{}{:04d}{:03d}{:03d}{}'.format(
-                        self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
+                            self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
                     filename = os.path.join(path, subfolder, filen)
                     self.ext = ext.lower()
                     self.path = path
                     if set is None:
                         self.setFile = -1
                     else:
                         self.setFile = set - 1
                     self.blocksPerFile = blocksPerFile

setup.py +1 -1

                   install_requires=[
                       "scipy >= 0.14.0",
                       "h5py >= 2.2.1",
-                      "matplotlib >= 1.4.2",
+                      "matplotlib >= 2.0.0",
                       "pyfits >= 3.4",
                       "paramiko >= 2.1.2",
                       "paho-mqtt >= 1.2",

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages