From 4bbef2dc4ffc5b2e743f8b549e4954ca53ec19e5 2017-11-03 22:49:20 From: Juan C. Espinoza Date: 2017-11-03 22:49:20 Subject: [PATCH] set new matplotlib version in setup, improved searchFiles for multiprocessing --- diff --git a/schainpy/controller.py b/schainpy/controller.py index 4b3039e..a34b86b 100644 --- a/schainpy/controller.py +++ b/schainpy/controller.py @@ -34,6 +34,8 @@ def MPProject(project, n=cpu_count()): op = rconf.getOperationObj('run') dt1 = op.getParameterValue('startDate') dt2 = op.getParameterValue('endDate') + tm1 = op.getParameterValue('startTime') + tm2 = op.getParameterValue('endTime') days = (dt2 - dt1).days for day in range(days + 1): @@ -46,11 +48,13 @@ def MPProject(project, n=cpu_count()): paths, files = reader.searchFilesOffLine(path=rconf.path, startDate=dt, endDate=dt, + startTime=tm1, + endTime=tm2, ext=DTYPES[rconf.datatype]) nFiles = len(files) if nFiles == 0: continue - skip = int(math.ceil(nFiles / n)) + skip = int(math.ceil(nFiles / n)) while nFiles > cursor * skip: rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor, skip=skip) @@ -962,7 +966,7 @@ class Project(Process): print '*' * 60 print self.id = str(id) - self.description = description + self.description = description def update(self, name, description): @@ -1270,7 +1274,7 @@ class Project(Process): def run(self): log.success('Starting {}'.format(self.name)) - + self.start_time = time.time() self.createObjects() self.connectObjects() @@ -1314,4 +1318,6 @@ class Project(Process): procUnitConfObj = self.procUnitConfObjDict[procKey] procUnitConfObj.close() - log.success('{} finished'.format(self.name)) + log.success('{} finished (time: {}s)'.format( + self.name, + time.time()-self.start_time)) diff --git a/schainpy/model/data/jroheaderIO.py b/schainpy/model/data/jroheaderIO.py index c85ae5d..59773a8 100644 --- a/schainpy/model/data/jroheaderIO.py +++ b/schainpy/model/data/jroheaderIO.py @@ -493,8 +493,8 @@ class RadarControllerHeader(Header): nBaud.tofile(fp) code1 = (self.code + 1.0)/2. - for ic in range(self.nCode): - tempx = numpy.zeros(numpy.ceil(self.nBaud/32.)) + for ic in range(self.nCode): + tempx = numpy.zeros(int(numpy.ceil(self.nBaud/32.))) start = 0 end = 32 for i in range(len(tempx)): diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index 90aab67..04e9232 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -22,6 +22,7 @@ except: from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width +from schainpy.utils import log LOCALTIME = True @@ -440,7 +441,7 @@ def isRadarFolder(folder): def isRadarFile(file): - try: + try: year = int(file[1:5]) doy = int(file[5:8]) set = int(file[8:11]) @@ -451,10 +452,10 @@ def isRadarFile(file): def getDateFromRadarFile(file): - try: + try: year = int(file[1:5]) doy = int(file[5:8]) - set = int(file[8:11]) + set = int(file[8:11]) except: return None @@ -649,26 +650,13 @@ class JRODataReader(JRODataIO): filenameList = [] datetimeList = [] - + for thisPath in pathList: fileList = glob.glob1(thisPath, "*%s" % ext) fileList.sort() - skippedFileList = [] - - if cursor is not None and skip is not None: - - if skip == 0: - skippedFileList = [] - else: - skippedFileList = fileList[cursor * - skip: cursor * skip + skip] - - else: - skippedFileList = fileList - - for file in skippedFileList: + for file in fileList: filename = os.path.join(thisPath, file) @@ -684,12 +672,15 @@ class JRODataReader(JRODataIO): filenameList.append(filename) datetimeList.append(thisDatetime) + if cursor is not None and skip is not None: + filenameList = filenameList[cursor * skip:cursor * skip + skip] + datetimeList = datetimeList[cursor * skip:cursor * skip + skip] + if not(filenameList): print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path) return [], [] - print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime) - print + print "[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime) # for i in range(len(filenameList)): # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) @@ -1743,9 +1734,9 @@ class JRODataWriter(JRODataIO): if self.dataOut.datatime.date() > self.fileDate: setFile = 0 self.nTotalBlocks = 0 - - filen = '%s%4.4d%3.3d%3.3d%s' % ( - self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) + + filen = '{}{:04d}{:03d}{:03d}{}'.format( + self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext) filename = os.path.join(path, subfolder, filen) @@ -1796,11 +1787,11 @@ class JRODataWriter(JRODataIO): self.ext = ext.lower() self.path = path - + if set is None: self.setFile = -1 else: - self.setFile = set - 1 + self.setFile = set - 1 self.blocksPerFile = blocksPerFile diff --git a/setup.py b/setup.py index 8b9d706..4578386 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup(name="schainpy", install_requires=[ "scipy >= 0.14.0", "h5py >= 2.2.1", - "matplotlib >= 1.4.2", + "matplotlib >= 2.0.0", "pyfits >= 3.4", "paramiko >= 2.1.2", "paho-mqtt >= 1.2",