diff --git a/schainpy/model/jrodata.py b/schainpy/model/jrodata.py index f2b6595..ca17ee8 100644 --- a/schainpy/model/jrodata.py +++ b/schainpy/model/jrodata.py @@ -572,3 +572,53 @@ class SpectraHeis(JROData): self.utctime = None self.blocksize = None + +class Fits: + + def __init__(self): + self.useLocalTime = False + self.utctime = None + self.timeZone = None + self.ltctime = None + self.timeInterval = None + self.header = None + self.data_header = None + self.data = None + self.datatime = None + self.flagNoData = False + self.expName = '' + self.nChannels = None + self.nSamples = None + self.dataBlocksPerFile = None + self.comments = '' + + + def getltctime(self): + + if self.useLocalTime: + return self.utctime - self.timeZone*60 + + return self.utctime + + def getDatatime(self): + + datatime = datetime.datetime.utcfromtimestamp(self.ltctime) + return datatime + + def getTimeRange(self): + + datatime = [] + + datatime.append(self.ltctime) + datatime.append(self.ltctime + self.timeInterval) + + datatime = numpy.array(datatime) + + return datatime + + def isEmpty(self): + + return self.flagNoData + + datatime = property(getDatatime, "I'm the 'datatime' property") + ltctime = property(getltctime, "I'm the 'ltctime' property") \ No newline at end of file diff --git a/schainpy/model/jrodataIO.py b/schainpy/model/jrodataIO.py index 6c524de..465ffa5 100644 --- a/schainpy/model/jrodataIO.py +++ b/schainpy/model/jrodataIO.py @@ -2919,3 +2919,395 @@ class FitsWriter(Operation): self.putData() +class FitsReader(ProcessingUnit): + + __TIMEZONE = time.timezone + + expName = None + datetimestr = None + utc = None + nChannels = None + nSamples = None + dataBlocksPerFile = None + comments = None + lastUTTime = None + header_dict = None + data = None + data_header_dict = None + + def __init__(self): + self.isConfig = False + self.ext = '.fits' + self.setFile = 0 + self.flagNoMoreFiles = 0 + self.flagIsNewFile = 1 + self.flagTimeBlock = None + self.fileIndex = None + self.filename = None + self.fileSize = None + self.fitsObj = None + self.nReadBlocks = 0 + self.nTotalBlocks = 0 + self.dataOut = self.createObjByDefault() + self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup() + self.blockIndex = 1 + + def createObjByDefault(self): + + dataObj = Fits() + + return dataObj + + def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False): + try: + fitsObj = pyfits.open(filename,'readonly') + except: + raise IOError, "The file %s can't be opened" %(filename) + + header = fitsObj[0].header + struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S") + utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS + + ltc = utc + if useLocalTime: + ltc -= time.timezone + thisDatetime = datetime.datetime.utcfromtimestamp(ltc) + thisTime = thisDatetime.time() + + if not ((startTime <= thisTime) and (endTime > thisTime)): + return None + + return thisDatetime + + def __setNextFileOnline(self): + raise ValueError, "No implemented" + + def __setNextFileOffline(self): + idFile = self.fileIndex + + while (True): + idFile += 1 + if not(idFile < len(self.filenameList)): + self.flagNoMoreFiles = 1 + print "No more Files" + return 0 + + filename = self.filenameList[idFile] + +# if not(self.__verifyFile(filename)): +# continue + + fileSize = os.path.getsize(filename) + fitsObj = pyfits.open(filename,'readonly') + break + + self.flagIsNewFile = 1 + self.fileIndex = idFile + self.filename = filename + self.fileSize = fileSize + self.fitsObj = fitsObj + + print "Setting the file: %s"%self.filename + + return 1 + + def readHeader(self): + headerObj = self.fitsObj[0] + + self.header_dict = headerObj.header + self.expName = headerObj.header['EXPNAME'] + self.datetimestr = headerObj.header['DATETIME'] + struct_time = time.strptime(headerObj.header['DATETIME'], "%b %d %Y %H:%M:%S") +# self.utc = time.mktime(struct_time) - self.__TIMEZONE + self.nChannels = headerObj.header['NCHANNEL'] + self.nSamples = headerObj.header['NSAMPLE'] + self.dataBlocksPerFile = headerObj.header['NBLOCK'] + self.comments = headerObj.header['COMMENT'] + + + def setNextFile(self): + + if self.online: + newFile = self.__setNextFileOnline() + else: + newFile = self.__setNextFileOffline() + + if not(newFile): + return 0 + + self.readHeader() + + self.nReadBlocks = 0 + self.blockIndex = 1 + return 1 + + def __searchFilesOffLine(self, + path, + startDate, + endDate, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + set=None, + expLabel='', + ext='.fits', + walk=True): + + pathList = [] + + if not walk: + pathList.append(path) + + else: + dirList = [] + for thisPath in os.listdir(path): + if not os.path.isdir(os.path.join(path,thisPath)): + continue + if not isDoyFolder(thisPath): + continue + + dirList.append(thisPath) + + if not(dirList): + return None, None + + thisDate = startDate + + while(thisDate <= endDate): + year = thisDate.timetuple().tm_year + doy = thisDate.timetuple().tm_yday + + matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*') + if len(matchlist) == 0: + thisDate += datetime.timedelta(1) + continue + for match in matchlist: + pathList.append(os.path.join(path,match,expLabel)) + + thisDate += datetime.timedelta(1) + + if pathList == []: + print "Any folder was found for the date range: %s-%s" %(startDate, endDate) + return None, None + + print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate) + + filenameList = [] + datetimeList = [] + + for i in range(len(pathList)): + + thisPath = pathList[i] + + fileList = glob.glob1(thisPath, "*%s" %ext) + fileList.sort() + + for file in fileList: + + filename = os.path.join(thisPath,file) + thisDatetime = self.isFileinThisTime(filename, startTime, endTime, useLocalTime=True) + + if not(thisDatetime): + continue + + filenameList.append(filename) + datetimeList.append(thisDatetime) + + if not(filenameList): + print "Any file was found for the time range %s - %s" %(startTime, endTime) + return None, None + + print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime) + print + + for i in range(len(filenameList)): + print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) + + self.filenameList = filenameList + self.datetimeList = datetimeList + + return pathList, filenameList + + def setup(self, path=None, + startDate=None, + endDate=None, + startTime=datetime.time(0,0,0), + endTime=datetime.time(23,59,59), + set=0, + expLabel = "", + ext = None, + online = False, + delay = 60, + walk = True): + + if path == None: + raise ValueError, "The path is not valid" + + if ext == None: + ext = self.ext + + if not(online): + print "Searching files in offline mode ..." + pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate, + startTime=startTime, endTime=endTime, + set=set, expLabel=expLabel, ext=ext, + walk=walk) + + if not(pathList): + print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path, + datetime.datetime.combine(startDate,startTime).ctime(), + datetime.datetime.combine(endDate,endTime).ctime()) + + sys.exit(-1) + + self.fileIndex = -1 + self.pathList = pathList + self.filenameList = filenameList + + self.online = online + self.delay = delay + ext = ext.lower() + self.ext = ext + + if not(self.setNextFile()): + if (startDate!=None) and (endDate!=None): + print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) + elif startDate != None: + print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()) + else: + print "No files" + + sys.exit(-1) + + + + def readBlock(self): + dataObj = self.fitsObj[self.blockIndex] + + self.data = dataObj.data + self.data_header_dict = dataObj.header + self.utc = self.data_header_dict['UTCTIME'] + + self.flagIsNewFile = 0 + self.blockIndex += 1 + self.nTotalBlocks += 1 + self.nReadBlocks += 1 + + return 1 + + def __jumpToLastBlock(self): + raise ValueError, "No implemented" + + def __waitNewBlock(self): + """ + Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. + + Si el modo de lectura es OffLine siempre retorn 0 + """ + if not self.online: + return 0 + + if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile): + return 0 + + currentPointer = self.fp.tell() + + neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize + + for nTries in range( self.nTries ): + + self.fp.close() + self.fp = open( self.filename, 'rb' ) + self.fp.seek( currentPointer ) + + self.fileSize = os.path.getsize( self.filename ) + currentSize = self.fileSize - currentPointer + + if ( currentSize >= neededSize ): + self.__rdBasicHeader() + return 1 + + print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) + time.sleep( self.delay ) + + + return 0 + + def __setNewBlock(self): + + if self.online: + self.__jumpToLastBlock() + + if self.flagIsNewFile: + return 1 + + self.lastUTTime = self.utc + + if self.online: + if self.__waitNewBlock(): + return 1 + + if self.nReadBlocks < self.dataBlocksPerFile: + return 1 + + if not(self.setNextFile()): + return 0 + + deltaTime = self.utc - self.lastUTTime + + self.flagTimeBlock = 0 + + if deltaTime > self.maxTimeStep: + self.flagTimeBlock = 1 + + return 1 + + + def readNextBlock(self): + if not(self.__setNewBlock()): + return 0 + + if not(self.readBlock()): + return 0 + + return 1 + + + def getData(self): + + if self.flagNoMoreFiles: + self.dataOut.flagNoData = True + print 'Process finished' + return 0 + + self.flagTimeBlock = 0 + self.flagIsNewBlock = 0 + + if not(self.readNextBlock()): + return 0 + + if self.data == None: + self.dataOut.flagNoData = True + return 0 + + self.dataOut.data = self.data + self.dataOut.data_header = self.data_header_dict + self.dataOut.utctime = self.utc + + self.dataOut.header = self.header_dict + self.dataOut.expName = self.expName + self.dataOut.nChannels = self.nChannels + self.dataOut.nSamples = self.nSamples + self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile + self.dataOut.comments = self.comments + + self.dataOut.flagNoData = False + + return self.dataOut.data + + def run(self, **kwargs): + + if not(self.isConfig): + self.setup(**kwargs) + self.isConfig = True + + self.getData() \ No newline at end of file diff --git a/schainpy/test/FitsExp.py b/schainpy/test/FitsExp.py new file mode 100644 index 0000000..72c5720 --- /dev/null +++ b/schainpy/test/FitsExp.py @@ -0,0 +1,45 @@ +import os, sys + +path = os.path.split(os.getcwd())[0] +sys.path.append(path) + +from controller import * + +desc = "FITS Test" +filename = "fitsexp.xml" + +controllerObj = Project() + +controllerObj.setup(id = '191', name='test01', description=desc) + +readUnitConfObj = controllerObj.addReadUnit(datatype='Fits', + path='/Users/dsuarez/Remote/d2013043', + startDate='2013/02/06', + endDate='2013/12/31', + startTime='00:30:00', + endTime='17:40:59', + online=0, + delay=3, + walk=0) + +#procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId()) + +#procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraHeis', inputId=procUnitConfObj0.getId()) +#procUnitConfObj1.addParameter(name='timeInterval', value='5', format='int') + +#opObj11 = procUnitConfObj1.addOperation(name='IncohInt4SpectraHeis', optype='other') +#opObj11.addParameter(name='timeInterval', value='1', format='float') + + + + +print "Escribiendo el archivo XML" +controllerObj.writeXml(filename) +print "Leyendo el archivo XML" +controllerObj.readXml(filename) + +controllerObj.createObjects() +controllerObj.connectObjects() +controllerObj.run() + + \ No newline at end of file