jroIO_heispectra.py
848 lines
| 24.7 KiB
| text/x-python
|
PythonLexer
Daniel Valdez
|
r487 | ''' | ||
Miguel Valdez
|
r568 | Created on Jul 3, 2014 | ||
|
Daniel Valdez
|
r487 | |||
|
Miguel Valdez
|
r568 | @author: roj-idl71 | ||
|
Daniel Valdez
|
r487 | ''' | ||
| import os, sys | ||||
| import time, datetime | ||||
| import numpy | ||||
| import fnmatch | ||||
| import glob | ||||
|
Miguel Valdez
|
r664 | from time import sleep | ||
|
Miguel Valdez
|
r568 | |||
|
Daniel Valdez
|
r487 | try: | ||
| import pyfits | ||||
|
Miguel Valdez
|
r664 | except ImportError, e: | ||
| print "Fits data cannot be used. Install pyfits module" | ||||
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | from xml.etree.ElementTree import ElementTree | ||
|
Miguel Valdez
|
r589 | from jroIO_base import isRadarFolder, isNumber | ||
|
Miguel Valdez
|
r664 | from schainpy.model.data.jrodata import Fits | ||
|
Miguel Valdez
|
r568 | from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit | ||
|
José Chávez
|
r897 | |||
| class PyFits(object): | ||||
|
Miguel Valdez
|
r568 | name=None | ||
| format=None | ||||
| array =None | ||||
| data =None | ||||
| thdulist=None | ||||
| prihdr=None | ||||
| hdu=None | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | def __init__(self): | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | pass | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | def setColF(self,name,format,array): | ||
| self.name=name | ||||
| self.format=format | ||||
| self.array=array | ||||
| a1=numpy.array([self.array],dtype=numpy.float32) | ||||
| self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1) | ||||
| return self.col1 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | # def setColP(self,name,format,data): | ||
| # self.name=name | ||||
| # self.format=format | ||||
| # self.data=data | ||||
| # a2=numpy.array([self.data],dtype=numpy.float32) | ||||
| # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2) | ||||
| # return self.col2 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | |||
| def writeData(self,name,format,data): | ||||
| self.name=name | ||||
| self.format=format | ||||
| self.data=data | ||||
| a2=numpy.array([self.data],dtype=numpy.float32) | ||||
| self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2) | ||||
| return self.col2 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | def cFImage(self,idblock,year,month,day,hour,minute,second): | ||
| self.hdu= pyfits.PrimaryHDU(idblock) | ||||
| self.hdu.header.set("Year",year) | ||||
| self.hdu.header.set("Month",month) | ||||
| self.hdu.header.set("Day",day) | ||||
| self.hdu.header.set("Hour",hour) | ||||
| self.hdu.header.set("Minute",minute) | ||||
| self.hdu.header.set("Second",second) | ||||
|
José Chávez
|
r897 | return self.hdu | ||
|
Miguel Valdez
|
r568 | |||
| def Ctable(self,colList): | ||||
| self.cols=pyfits.ColDefs(colList) | ||||
| self.tbhdu = pyfits.new_table(self.cols) | ||||
| return self.tbhdu | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | def CFile(self,hdu,tbhdu): | ||
| self.thdulist=pyfits.HDUList([hdu,tbhdu]) | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | def wFile(self,filename): | ||
| if os.path.isfile(filename): | ||||
| os.remove(filename) | ||||
|
José Chávez
|
r897 | self.thdulist.writeto(filename) | ||
|
Miguel Valdez
|
r568 | |||
|
Daniel Valdez
|
r487 | |||
| class ParameterConf: | ||||
| ELEMENTNAME = 'Parameter' | ||||
| def __init__(self): | ||||
| self.name = '' | ||||
| self.value = '' | ||||
| def readXml(self, parmElement): | ||||
| self.name = parmElement.get('name') | ||||
| self.value = parmElement.get('value') | ||||
| def getElementName(self): | ||||
| return self.ELEMENTNAME | ||||
|
Miguel Valdez
|
r664 | class Metadata(object): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def __init__(self, filename): | ||
| self.parmConfObjList = [] | ||||
| self.readXml(filename) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def readXml(self, filename): | ||
| self.projectElement = None | ||||
| self.procUnitConfObjDict = {} | ||||
| self.projectElement = ElementTree().parse(filename) | ||||
|
José Chávez
|
r897 | self.project = self.projectElement.tag | ||
|
Daniel Valdez
|
r487 | |||
| parmElementList = self.projectElement.getiterator(ParameterConf().getElementName()) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | for parmElement in parmElementList: | ||
| parmConfObj = ParameterConf() | ||||
| parmConfObj.readXml(parmElement) | ||||
| self.parmConfObjList.append(parmConfObj) | ||||
| class FitsWriter(Operation): | ||||
|
José Chávez
|
r897 | def __init__(self, **kwargs): | ||
| Operation.__init__(self, **kwargs) | ||||
|
Daniel Valdez
|
r487 | self.isConfig = False | ||
| self.dataBlocksPerFile = None | ||||
| self.blockIndex = 0 | ||||
| self.flagIsNewFile = 1 | ||||
| self.fitsObj = None | ||||
| self.optchar = 'P' | ||||
| self.ext = '.fits' | ||||
| self.setFile = 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | def setFitsHeader(self, dataOut, metadatafile=None): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | header_data = pyfits.PrimaryHDU() | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | header_data.header['EXPNAME'] = "RADAR DATA" | ||
| header_data.header['DATATYPE'] = "SPECTRA" | ||||
| header_data.header['COMMENT'] = "" | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | if metadatafile: | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | metadata4fits = Metadata(metadatafile) | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | for parameter in metadata4fits.parmConfObjList: | ||
| parm_name = parameter.name | ||||
| parm_value = parameter.value | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | header_data.header[parm_name] = parm_value | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple()) | ||
| header_data.header['CHANNELLIST'] = str(dataOut.channelList) | ||||
| header_data.header['NCHANNELS'] = dataOut.nChannels | ||||
| #header_data.header['HEIGHTS'] = dataOut.heightList | ||||
| header_data.header['NHEIGHTS'] = dataOut.nHeights | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | header_data.header['IPPSECONDS'] = dataOut.ippSeconds | ||
| header_data.header['NCOHINT'] = dataOut.nCohInt | ||||
| header_data.header['NINCOHINT'] = dataOut.nIncohInt | ||||
| header_data.header['TIMEZONE'] = dataOut.timeZone | ||||
| header_data.header['NBLOCK'] = self.blockIndex | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | header_data.writeto(self.filename) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.addExtension(dataOut.heightList,'HEIGHTLIST') | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.path = path | ||
| self.dataOut = dataOut | ||||
| self.metadatafile = metadatafile | ||||
| self.dataBlocksPerFile = dataBlocksPerFile | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def open(self): | ||
| self.fitsObj = pyfits.open(self.filename, mode='update') | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def addExtension(self, data, tagname): | ||
| self.open() | ||||
| extension = pyfits.ImageHDU(data=data, name=tagname) | ||||
| #extension.header['TAG'] = tagname | ||||
| self.fitsObj.append(extension) | ||||
| self.write() | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def addData(self, data): | ||
| self.open() | ||||
| extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE']) | ||||
| extension.header['UTCTIME'] = self.dataOut.utctime | ||||
| self.fitsObj.append(extension) | ||||
| self.blockIndex += 1 | ||||
| self.fitsObj[0].header['NBLOCK'] = self.blockIndex | ||||
| self.write() | ||||
| def write(self): | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.fitsObj.flush(verbose=True) | ||
| self.fitsObj.close() | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def setNextFile(self): | ||
| ext = self.ext | ||||
| path = self.path | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | timeTuple = time.localtime( self.dataOut.utctime) | ||
| subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | ||||
| fullpath = os.path.join( path, subfolder ) | ||||
| if not( os.path.exists(fullpath) ): | ||||
| os.mkdir(fullpath) | ||||
| self.setFile = -1 #inicializo mi contador de seteo | ||||
| else: | ||||
| filesList = os.listdir( fullpath ) | ||||
| if len( filesList ) > 0: | ||||
| filesList = sorted( filesList, key=str.lower ) | ||||
| filen = filesList[-1] | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if isNumber( filen[8:11] ): | ||
| self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | ||||
|
José Chávez
|
r897 | else: | ||
|
Daniel Valdez
|
r487 | self.setFile = -1 | ||
| else: | ||||
| self.setFile = -1 #inicializo mi contador de seteo | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | setFile = self.setFile | ||
| setFile += 1 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, | ||
|
Daniel Valdez
|
r487 | timeTuple.tm_year, | ||
| timeTuple.tm_yday, | ||||
| setFile, | ||||
| ext ) | ||||
|
Miguel Valdez
|
r568 | filename = os.path.join( path, subfolder, thisFile ) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.blockIndex = 0 | ||
| self.filename = filename | ||||
| self.setFile = setFile | ||||
| self.flagIsNewFile = 1 | ||||
| print 'Writing the file: %s'%self.filename | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.setFitsHeader(self.dataOut, self.metadatafile) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return 1 | ||
| def writeBlock(self): | ||||
|
José Chávez
|
r897 | self.addData(self.dataOut.data_spc) | ||
|
Daniel Valdez
|
r487 | self.flagIsNewFile = 0 | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def __setNewBlock(self): | ||
| if self.flagIsNewFile: | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if self.blockIndex < self.dataBlocksPerFile: | ||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not( self.setNextFile() ): | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return 1 | ||
| def writeNextBlock(self): | ||||
| if not( self.__setNewBlock() ): | ||||
| return 0 | ||||
| self.writeBlock() | ||||
|
José Chávez
|
r897 | return 1 | ||
| def putData(self): | ||||
|
Daniel Valdez
|
r487 | if self.flagIsNewFile: | ||
| self.setNextFile() | ||||
| self.writeNextBlock() | ||||
|
José Chávez
|
r897 | |||
|
José Chávez
|
r954 | def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs): | ||
|
Daniel Valdez
|
r487 | if not(self.isConfig): | ||
|
José Chávez
|
r954 | self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs) | ||
|
Daniel Valdez
|
r487 | self.isConfig = True | ||
| self.putData() | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | class FitsReader(ProcessingUnit): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | # __TIMEZONE = time.timezone | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | expName = None | ||
| datetimestr = None | ||||
| utc = None | ||||
| nChannels = None | ||||
| nSamples = None | ||||
| dataBlocksPerFile = None | ||||
|
José Chávez
|
r897 | comments = None | ||
|
Daniel Valdez
|
r487 | lastUTTime = None | ||
| header_dict = None | ||||
| data = None | ||||
| data_header_dict = None | ||||
|
José Chávez
|
r897 | |||
| def __init__(self, **kwargs): | ||||
| ProcessingUnit.__init__(self, **kwargs) | ||||
|
Daniel Valdez
|
r487 | self.isConfig = False | ||
| self.ext = '.fits' | ||||
| self.setFile = 0 | ||||
| self.flagNoMoreFiles = 0 | ||||
| self.flagIsNewFile = 1 | ||||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = None | ||
|
Daniel Valdez
|
r487 | self.fileIndex = None | ||
| self.filename = None | ||||
| self.fileSize = None | ||||
| self.fitsObj = None | ||||
| self.timeZone = None | ||||
| self.nReadBlocks = 0 | ||||
| self.nTotalBlocks = 0 | ||||
| self.dataOut = self.createObjByDefault() | ||||
| self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup() | ||||
|
José Chávez
|
r897 | self.blockIndex = 1 | ||
|
Daniel Valdez
|
r487 | def createObjByDefault(self): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | dataObj = Fits() | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return dataObj | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False): | ||
| try: | ||||
| fitsObj = pyfits.open(filename,'readonly') | ||||
| except: | ||||
|
Miguel Valdez
|
r684 | print "File %s can't be opened" %(filename) | ||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | header = fitsObj[0].header | ||
| struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S") | ||||
| utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | ltc = utc | ||
| if useLocalTime: | ||||
| ltc -= time.timezone | ||||
| thisDatetime = datetime.datetime.utcfromtimestamp(ltc) | ||||
| thisTime = thisDatetime.time() | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not ((startTime <= thisTime) and (endTime > thisTime)): | ||
| return None | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return thisDatetime | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def __setNextFileOnline(self): | ||
|
Miguel Valdez
|
r684 | raise NotImplementedError | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def __setNextFileOffline(self): | ||
| idFile = self.fileIndex | ||||
| while (True): | ||||
| idFile += 1 | ||||
| if not(idFile < len(self.filenameList)): | ||||
| self.flagNoMoreFiles = 1 | ||||
| print "No more Files" | ||||
| return 0 | ||||
| filename = self.filenameList[idFile] | ||||
| # if not(self.__verifyFile(filename)): | ||||
| # continue | ||||
| fileSize = os.path.getsize(filename) | ||||
| fitsObj = pyfits.open(filename,'readonly') | ||||
| break | ||||
| self.flagIsNewFile = 1 | ||||
| self.fileIndex = idFile | ||||
| self.filename = filename | ||||
| self.fileSize = fileSize | ||||
| self.fitsObj = fitsObj | ||||
| self.blockIndex = 0 | ||||
| print "Setting the file: %s"%self.filename | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r587 | def __setValuesFromHeader(self): | ||
|
José Chávez
|
r897 | |||
| self.dataOut.header = self.header_dict | ||||
|
Miguel Valdez
|
r587 | self.dataOut.expName = self.expName | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r587 | self.dataOut.timeZone = self.timeZone | ||
| self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile | ||||
|
José Chávez
|
r897 | self.dataOut.comments = self.comments | ||
|
Miguel Valdez
|
r587 | # self.dataOut.timeInterval = self.timeInterval | ||
| self.dataOut.channelList = self.channelList | ||||
| self.dataOut.heightList = self.heightList | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r587 | self.dataOut.nCohInt = self.nCohInt | ||
| self.dataOut.nIncohInt = self.nIncohInt | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | self.dataOut.ippSeconds = self.ippSeconds | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def readHeader(self): | ||
| headerObj = self.fitsObj[0] | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.header_dict = headerObj.header | ||
| if 'EXPNAME' in headerObj.header.keys(): | ||||
| self.expName = headerObj.header['EXPNAME'] | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if 'DATATYPE' in headerObj.header.keys(): | ||
| self.dataType = headerObj.header['DATATYPE'] | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.datetimestr = headerObj.header['DATETIME'] | ||
| channelList = headerObj.header['CHANNELLIST'] | ||||
| channelList = channelList.split('[') | ||||
| channelList = channelList[1].split(']') | ||||
| channelList = channelList[0].split(',') | ||||
| channelList = [int(ch) for ch in channelList] | ||||
| self.channelList = channelList | ||||
| self.nChannels = headerObj.header['NCHANNELS'] | ||||
| self.nHeights = headerObj.header['NHEIGHTS'] | ||||
| self.ippSeconds = headerObj.header['IPPSECONDS'] | ||||
| self.nCohInt = headerObj.header['NCOHINT'] | ||||
| self.nIncohInt = headerObj.header['NINCOHINT'] | ||||
| self.dataBlocksPerFile = headerObj.header['NBLOCK'] | ||||
| self.timeZone = headerObj.header['TIMEZONE'] | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r527 | # self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if 'COMMENT' in headerObj.header.keys(): | ||
| self.comments = headerObj.header['COMMENT'] | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.readHeightList() | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def readHeightList(self): | ||
| self.blockIndex = self.blockIndex + 1 | ||||
| obj = self.fitsObj[self.blockIndex] | ||||
| self.heightList = obj.data | ||||
| self.blockIndex = self.blockIndex + 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def readExtension(self): | ||
| obj = self.fitsObj[self.blockIndex] | ||||
| self.heightList = obj.data | ||||
| self.blockIndex = self.blockIndex + 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def setNextFile(self): | ||
| if self.online: | ||||
| newFile = self.__setNextFileOnline() | ||||
| else: | ||||
| newFile = self.__setNextFileOffline() | ||||
| if not(newFile): | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.readHeader() | ||
|
Miguel Valdez
|
r587 | self.__setValuesFromHeader() | ||
|
Daniel Valdez
|
r487 | self.nReadBlocks = 0 | ||
| # self.blockIndex = 1 | ||||
| return 1 | ||||
Juan C. Espinoza
|
r1052 | def searchFilesOffLine(self, | ||
|
Daniel Valdez
|
r487 | path, | ||
| startDate, | ||||
| endDate, | ||||
| startTime=datetime.time(0,0,0), | ||||
| endTime=datetime.time(23,59,59), | ||||
| set=None, | ||||
| expLabel='', | ||||
| ext='.fits', | ||||
| walk=True): | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | pathList = [] | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not walk: | ||
| pathList.append(path) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | else: | ||
| dirList = [] | ||||
| for thisPath in os.listdir(path): | ||||
| if not os.path.isdir(os.path.join(path,thisPath)): | ||||
| continue | ||||
|
Miguel Valdez
|
r589 | if not isRadarFolder(thisPath): | ||
|
Daniel Valdez
|
r487 | continue | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | dirList.append(thisPath) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not(dirList): | ||
| return None, None | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | thisDate = startDate | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | while(thisDate <= endDate): | ||
| year = thisDate.timetuple().tm_year | ||||
| doy = thisDate.timetuple().tm_yday | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*') | ||
| if len(matchlist) == 0: | ||||
| thisDate += datetime.timedelta(1) | ||||
| continue | ||||
| for match in matchlist: | ||||
| pathList.append(os.path.join(path,match,expLabel)) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | thisDate += datetime.timedelta(1) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if pathList == []: | ||
| print "Any folder was found for the date range: %s-%s" %(startDate, endDate) | ||||
| return None, None | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | filenameList = [] | ||
| datetimeList = [] | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | for i in range(len(pathList)): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | thisPath = pathList[i] | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | fileList = glob.glob1(thisPath, "*%s" %ext) | ||
| fileList.sort() | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | for thisFile in fileList: | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | filename = os.path.join(thisPath,thisFile) | ||
|
Daniel Valdez
|
r487 | thisDatetime = self.isFileinThisTime(filename, startTime, endTime) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not(thisDatetime): | ||
| continue | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | filenameList.append(filename) | ||
| datetimeList.append(thisDatetime) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not(filenameList): | ||
| print "Any file was found for the time range %s - %s" %(startTime, endTime) | ||||
| return None, None | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | for i in range(len(filenameList)): | ||
| print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()) | ||||
| self.filenameList = filenameList | ||||
| self.datetimeList = datetimeList | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return pathList, filenameList | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def setup(self, path=None, | ||
|
José Chávez
|
r897 | startDate=None, | ||
| endDate=None, | ||||
| startTime=datetime.time(0,0,0), | ||||
| endTime=datetime.time(23,59,59), | ||||
| set=0, | ||||
| expLabel = "", | ||||
| ext = None, | ||||
|
Daniel Valdez
|
r487 | online = False, | ||
| delay = 60, | ||||
| walk = True): | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if path == None: | ||
| raise ValueError, "The path is not valid" | ||||
| if ext == None: | ||||
| ext = self.ext | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not(online): | ||
| print "Searching files in offline mode ..." | ||||
|
Juan C. Espinoza
|
r1052 | pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, | ||
|
Daniel Valdez
|
r487 | startTime=startTime, endTime=endTime, | ||
| set=set, expLabel=expLabel, ext=ext, | ||||
| walk=walk) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not(pathList): | ||
| print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path, | ||||
| datetime.datetime.combine(startDate,startTime).ctime(), | ||||
| datetime.datetime.combine(endDate,endTime).ctime()) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | sys.exit(-1) | ||
| self.fileIndex = -1 | ||||
| self.pathList = pathList | ||||
| self.filenameList = filenameList | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.online = online | ||
| self.delay = delay | ||||
| ext = ext.lower() | ||||
| self.ext = ext | ||||
| if not(self.setNextFile()): | ||||
| if (startDate!=None) and (endDate!=None): | ||||
| print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()) | ||||
| elif startDate != None: | ||||
| print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()) | ||||
| else: | ||||
| print "No files" | ||||
| sys.exit(-1) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def readBlock(self): | ||
| dataObj = self.fitsObj[self.blockIndex] | ||||
| self.data = dataObj.data | ||||
| self.data_header_dict = dataObj.header | ||||
| self.utc = self.data_header_dict['UTCTIME'] | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.flagIsNewFile = 0 | ||
| self.blockIndex += 1 | ||||
| self.nTotalBlocks += 1 | ||||
| self.nReadBlocks += 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def __jumpToLastBlock(self): | ||
|
Miguel Valdez
|
r684 | raise NotImplementedError | ||
|
Daniel Valdez
|
r487 | |||
| def __waitNewBlock(self): | ||||
| """ | ||||
|
José Chávez
|
r897 | Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. | ||
|
Daniel Valdez
|
r487 | Si el modo de lectura es OffLine siempre retorn 0 | ||
| """ | ||||
| if not self.online: | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if (self.nReadBlocks >= self.dataBlocksPerFile): | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | currentPointer = self.fp.tell() | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | for nTries in range( self.nTries ): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.fp.close() | ||
| self.fp = open( self.filename, 'rb' ) | ||||
| self.fp.seek( currentPointer ) | ||||
| self.fileSize = os.path.getsize( self.filename ) | ||||
| currentSize = self.fileSize - currentPointer | ||||
| if ( currentSize >= neededSize ): | ||||
| self.__rdBasicHeader() | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1) | ||
|
Miguel Valdez
|
r568 | sleep( self.delay ) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return 0 | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def __setNewBlock(self): | ||
| if self.online: | ||||
| self.__jumpToLastBlock() | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if self.flagIsNewFile: | ||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.lastUTTime = self.utc | ||
| if self.online: | ||||
| if self.__waitNewBlock(): | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if self.nReadBlocks < self.dataBlocksPerFile: | ||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not(self.setNextFile()): | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
| deltaTime = self.utc - self.lastUTTime | ||||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = 0 | ||
|
Daniel Valdez
|
r487 | |||
| if deltaTime > self.maxTimeStep: | ||||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = 1 | ||
|
Daniel Valdez
|
r487 | |||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def readNextBlock(self): | ||
| if not(self.__setNewBlock()): | ||||
| return 0 | ||||
| if not(self.readBlock()): | ||||
| return 0 | ||||
| return 1 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | def printInfo(self): | ||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | pass | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def getData(self): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if self.flagNoMoreFiles: | ||
| self.dataOut.flagNoData = True | ||||
| print 'Process finished' | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = 0 | ||
|
Daniel Valdez
|
r487 | self.flagIsNewBlock = 0 | ||
| if not(self.readNextBlock()): | ||||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Miguel Valdez
|
r664 | if self.data is None: | ||
|
Daniel Valdez
|
r487 | self.dataOut.flagNoData = True | ||
| return 0 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.dataOut.data = self.data | ||
| self.dataOut.data_header = self.data_header_dict | ||||
| self.dataOut.utctime = self.utc | ||||
|
José Chávez
|
r897 | |||
| # self.dataOut.header = self.header_dict | ||||
|
Miguel Valdez
|
r587 | # self.dataOut.expName = self.expName | ||
| # self.dataOut.nChannels = self.nChannels | ||||
| # self.dataOut.timeZone = self.timeZone | ||||
| # self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile | ||||
|
José Chávez
|
r897 | # self.dataOut.comments = self.comments | ||
|
Miguel Valdez
|
r587 | # # self.dataOut.timeInterval = self.timeInterval | ||
| # self.dataOut.channelList = self.channelList | ||||
| # self.dataOut.heightList = self.heightList | ||||
|
Daniel Valdez
|
r487 | self.dataOut.flagNoData = False | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return self.dataOut.data | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def run(self, **kwargs): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not(self.isConfig): | ||
| self.setup(**kwargs) | ||||
| self.isConfig = True | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.getData() | ||
|
José Chávez
|
r897 | class SpectraHeisWriter(Operation): | ||
|
Daniel Valdez
|
r487 | # set = None | ||
| setFile = None | ||||
| idblock = None | ||||
| doypath = None | ||||
| subfolder = None | ||||
|
José Chávez
|
r897 | |||
| def __init__(self, **kwargs): | ||||
| Operation.__init__(self, **kwargs) | ||||
|
Miguel Valdez
|
r664 | self.wrObj = PyFits() | ||
|
José Chávez
|
r897 | # self.dataOut = dataOut | ||
|
Daniel Valdez
|
r487 | self.nTotalBlocks=0 | ||
| # self.set = None | ||||
| self.setFile = None | ||||
| self.idblock = 0 | ||||
| self.wrpath = None | ||||
| self.doypath = None | ||||
| self.subfolder = None | ||||
|
José Chávez
|
r897 | self.isConfig = False | ||
|
Daniel Valdez
|
r487 | def isNumber(str): | ||
| """ | ||||
| Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. | ||||
|
José Chávez
|
r897 | Excepciones: | ||
|
Daniel Valdez
|
r487 | Si un determinado string no puede ser convertido a numero | ||
| Input: | ||||
| str, string al cual se le analiza para determinar si convertible a un numero o no | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | Return: | ||
| True : si el string es uno numerico | ||||
| False : no es un string numerico | ||||
| """ | ||||
| try: | ||||
| float( str ) | ||||
| return True | ||||
| except: | ||||
|
José Chávez
|
r897 | return False | ||
|
Daniel Valdez
|
r487 | def setup(self, dataOut, wrpath): | ||
| if not(os.path.exists(wrpath)): | ||||
| os.mkdir(wrpath) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.wrpath = wrpath | ||
| # self.setFile = 0 | ||||
| self.dataOut = dataOut | ||||
| def putData(self): | ||||
| name= time.localtime( self.dataOut.utctime) | ||||
|
José Chávez
|
r897 | ext=".fits" | ||
|
Daniel Valdez
|
r487 | if self.doypath == None: | ||
| self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple())) | ||||
| self.doypath = os.path.join( self.wrpath, self.subfolder ) | ||||
| os.mkdir(self.doypath) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if self.setFile == None: | ||
| # self.set = self.dataOut.set | ||||
| self.setFile = 0 | ||||
| # if self.set != self.dataOut.set: | ||||
| ## self.set = self.dataOut.set | ||||
| # self.setFile = 0 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | #make the filename | ||
|
José Chávez
|
r897 | thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext) | ||
| filename = os.path.join(self.wrpath,self.subfolder, thisFile) | ||||
|
Daniel Valdez
|
r487 | |||
| idblock = numpy.array([self.idblock],dtype="int64") | ||||
|
José Chávez
|
r897 | header=self.wrObj.cFImage(idblock=idblock, | ||
|
Daniel Valdez
|
r487 | year=time.gmtime(self.dataOut.utctime).tm_year, | ||
| month=time.gmtime(self.dataOut.utctime).tm_mon, | ||||
|
José Chávez
|
r897 | day=time.gmtime(self.dataOut.utctime).tm_mday, | ||
|
Daniel Valdez
|
r487 | hour=time.gmtime(self.dataOut.utctime).tm_hour, | ||
|
José Chávez
|
r897 | minute=time.gmtime(self.dataOut.utctime).tm_min, | ||
|
Daniel Valdez
|
r487 | second=time.gmtime(self.dataOut.utctime).tm_sec) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | c=3E8 | ||
| deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0] | ||||
| freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000)) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | colList = [] | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | colList.append(colFreq) | ||
| nchannel=self.dataOut.nChannels | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | for i in range(nchannel): | ||
| col = self.wrObj.writeData(name="PCh"+str(i+1), | ||||
| format=str(self.dataOut.nFFTPoints)+'E', | ||||
| data=10*numpy.log10(self.dataOut.data_spc[i,:])) | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | colList.append(col) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | data=self.wrObj.Ctable(colList=colList) | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.wrObj.CFile(header,data) | ||
|
José Chávez
|
r897 | |||
| self.wrObj.wFile(filename) | ||||
|
Daniel Valdez
|
r487 | #update the setFile | ||
| self.setFile += 1 | ||||
| self.idblock += 1 | ||||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | return 1 | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | def run(self, dataOut, **kwargs): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | if not(self.isConfig): | ||
|
José Chávez
|
r897 | |||
|
Daniel Valdez
|
r487 | self.setup(dataOut, **kwargs) | ||
| self.isConfig = True | ||||
|
José Chávez
|
r897 | |||
| self.putData() | ||||