jroIO_heispectra.py
847 lines
| 24.8 KiB
| text/x-python
|
PythonLexer
Daniel Valdez
|
r487 | ''' | |
Miguel Valdez
|
r568 | Created on Jul 3, 2014 | |
|
Daniel Valdez
|
r487 | ||
|
Miguel Valdez
|
r568 | @author: roj-idl71 | |
|
Daniel Valdez
|
r487 | ''' | |
| import os, sys | |||
| import time, datetime | |||
| import numpy | |||
| import fnmatch | |||
| import glob | |||
|
Miguel Valdez
|
r664 | from time import sleep | |
|
Miguel Valdez
|
r568 | ||
|
Daniel Valdez
|
r487 | try: | |
| import pyfits | |||
George Yong
|
r1167 | except ImportError as e: | |
| print("Fits data cannot be used. Install pyfits module") | |||
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | from xml.etree.ElementTree import ElementTree | |
|
George Yong
|
r1167 | from .jroIO_base import isRadarFolder, isNumber | |
|
Miguel Valdez
|
r664 | from schainpy.model.data.jrodata import Fits | |
|
Miguel Valdez
|
r568 | from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit | |
|
José Chávez
|
r897 | ||
| class PyFits(object): | |||
|
Miguel Valdez
|
r568 | name=None | |
| format=None | |||
| array =None | |||
| data =None | |||
| thdulist=None | |||
| prihdr=None | |||
| hdu=None | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | def __init__(self): | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | pass | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | def setColF(self,name,format,array): | |
| self.name=name | |||
| self.format=format | |||
| self.array=array | |||
| a1=numpy.array([self.array],dtype=numpy.float32) | |||
| self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1) | |||
| return self.col1 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | # def setColP(self,name,format,data): | |
| # self.name=name | |||
| # self.format=format | |||
| # self.data=data | |||
| # a2=numpy.array([self.data],dtype=numpy.float32) | |||
| # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2) | |||
| # return self.col2 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | ||
| def writeData(self,name,format,data): | |||
| self.name=name | |||
| self.format=format | |||
| self.data=data | |||
| a2=numpy.array([self.data],dtype=numpy.float32) | |||
| self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2) | |||
| return self.col2 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | def cFImage(self,idblock,year,month,day,hour,minute,second): | |
| self.hdu= pyfits.PrimaryHDU(idblock) | |||
| self.hdu.header.set("Year",year) | |||
| self.hdu.header.set("Month",month) | |||
| self.hdu.header.set("Day",day) | |||
| self.hdu.header.set("Hour",hour) | |||
| self.hdu.header.set("Minute",minute) | |||
| self.hdu.header.set("Second",second) | |||
|
José Chávez
|
r897 | return self.hdu | |
|
Miguel Valdez
|
r568 | ||
| def Ctable(self,colList): | |||
| self.cols=pyfits.ColDefs(colList) | |||
| self.tbhdu = pyfits.new_table(self.cols) | |||
| return self.tbhdu | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | def CFile(self,hdu,tbhdu): | |
| self.thdulist=pyfits.HDUList([hdu,tbhdu]) | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | def wFile(self,filename): | |
| if os.path.isfile(filename): | |||
| os.remove(filename) | |||
|
José Chávez
|
r897 | self.thdulist.writeto(filename) | |
|
Miguel Valdez
|
r568 | ||
|
Daniel Valdez
|
r487 | ||
| class ParameterConf: | |||
| ELEMENTNAME = 'Parameter' | |||
| def __init__(self): | |||
| self.name = '' | |||
| self.value = '' | |||
| def readXml(self, parmElement): | |||
| self.name = parmElement.get('name') | |||
| self.value = parmElement.get('value') | |||
| def getElementName(self): | |||
| return self.ELEMENTNAME | |||
|
Miguel Valdez
|
r664 | class Metadata(object): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def __init__(self, filename): | |
| self.parmConfObjList = [] | |||
| self.readXml(filename) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def readXml(self, filename): | |
| self.projectElement = None | |||
| self.procUnitConfObjDict = {} | |||
| self.projectElement = ElementTree().parse(filename) | |||
|
José Chávez
|
r897 | self.project = self.projectElement.tag | |
|
Daniel Valdez
|
r487 | ||
| parmElementList = self.projectElement.getiterator(ParameterConf().getElementName()) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for parmElement in parmElementList: | |
| parmConfObj = ParameterConf() | |||
| parmConfObj.readXml(parmElement) | |||
| self.parmConfObjList.append(parmConfObj) | |||
| class FitsWriter(Operation): | |||
|
José Chávez
|
r897 | def __init__(self, **kwargs): | |
| Operation.__init__(self, **kwargs) | |||
|
Daniel Valdez
|
r487 | self.isConfig = False | |
| self.dataBlocksPerFile = None | |||
| self.blockIndex = 0 | |||
| self.flagIsNewFile = 1 | |||
| self.fitsObj = None | |||
| self.optchar = 'P' | |||
| self.ext = '.fits' | |||
| self.setFile = 0 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | def setFitsHeader(self, dataOut, metadatafile=None): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | header_data = pyfits.PrimaryHDU() | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | header_data.header['EXPNAME'] = "RADAR DATA" | |
| header_data.header['DATATYPE'] = "SPECTRA" | |||
| header_data.header['COMMENT'] = "" | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | if metadatafile: | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | metadata4fits = Metadata(metadatafile) | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | for parameter in metadata4fits.parmConfObjList: | |
| parm_name = parameter.name | |||
| parm_value = parameter.value | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | header_data.header[parm_name] = parm_value | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple()) | |
| header_data.header['CHANNELLIST'] = str(dataOut.channelList) | |||
| header_data.header['NCHANNELS'] = dataOut.nChannels | |||
| #header_data.header['HEIGHTS'] = dataOut.heightList | |||
| header_data.header['NHEIGHTS'] = dataOut.nHeights | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | header_data.header['IPPSECONDS'] = dataOut.ippSeconds | |
| header_data.header['NCOHINT'] = dataOut.nCohInt | |||
| header_data.header['NINCOHINT'] = dataOut.nIncohInt | |||
| header_data.header['TIMEZONE'] = dataOut.timeZone | |||
| header_data.header['NBLOCK'] = self.blockIndex | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | header_data.writeto(self.filename) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.addExtension(dataOut.heightList,'HEIGHTLIST') | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.path = path | |
| self.dataOut = dataOut | |||
| self.metadatafile = metadatafile | |||
| self.dataBlocksPerFile = dataBlocksPerFile | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def open(self): | |
| self.fitsObj = pyfits.open(self.filename, mode='update') | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def addExtension(self, data, tagname): | |
| self.open() | |||
| extension = pyfits.ImageHDU(data=data, name=tagname) | |||
| #extension.header['TAG'] = tagname | |||
| self.fitsObj.append(extension) | |||
| self.write() | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def addData(self, data): | |
| self.open() | |||
| extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE']) | |||
| extension.header['UTCTIME'] = self.dataOut.utctime | |||
| self.fitsObj.append(extension) | |||
| self.blockIndex += 1 | |||
| self.fitsObj[0].header['NBLOCK'] = self.blockIndex | |||
| self.write() | |||
| def write(self): | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.fitsObj.flush(verbose=True) | |
| self.fitsObj.close() | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def setNextFile(self): | |
| ext = self.ext | |||
| path = self.path | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | timeTuple = time.localtime( self.dataOut.utctime) | |
| subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday) | |||
| fullpath = os.path.join( path, subfolder ) | |||
| if not( os.path.exists(fullpath) ): | |||
| os.mkdir(fullpath) | |||
| self.setFile = -1 #inicializo mi contador de seteo | |||
| else: | |||
| filesList = os.listdir( fullpath ) | |||
| if len( filesList ) > 0: | |||
| filesList = sorted( filesList, key=str.lower ) | |||
| filen = filesList[-1] | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if isNumber( filen[8:11] ): | |
| self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file | |||
|
José Chávez
|
r897 | else: | |
|
Daniel Valdez
|
r487 | self.setFile = -1 | |
| else: | |||
| self.setFile = -1 #inicializo mi contador de seteo | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | setFile = self.setFile | |
| setFile += 1 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, | |
|
Daniel Valdez
|
r487 | timeTuple.tm_year, | |
| timeTuple.tm_yday, | |||
| setFile, | |||
| ext ) | |||
|
Miguel Valdez
|
r568 | filename = os.path.join( path, subfolder, thisFile ) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.blockIndex = 0 | |
| self.filename = filename | |||
| self.setFile = setFile | |||
| self.flagIsNewFile = 1 | |||
|
George Yong
|
r1167 | print('Writing the file: %s'%self.filename) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.setFitsHeader(self.dataOut, self.metadatafile) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
| def writeBlock(self): | |||
|
José Chávez
|
r897 | self.addData(self.dataOut.data_spc) | |
|
Daniel Valdez
|
r487 | self.flagIsNewFile = 0 | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def __setNewBlock(self): | |
| if self.flagIsNewFile: | |||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.blockIndex < self.dataBlocksPerFile: | |
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not( self.setNextFile() ): | |
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
| def writeNextBlock(self): | |||
| if not( self.__setNewBlock() ): | |||
| return 0 | |||
| self.writeBlock() | |||
|
José Chávez
|
r897 | return 1 | |
| def putData(self): | |||
|
Daniel Valdez
|
r487 | if self.flagIsNewFile: | |
| self.setNextFile() | |||
| self.writeNextBlock() | |||
|
José Chávez
|
r897 | ||
|
José Chávez
|
r954 | def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs): | |
|
Daniel Valdez
|
r487 | if not(self.isConfig): | |
|
José Chávez
|
r954 | self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs) | |
|
Daniel Valdez
|
r487 | self.isConfig = True | |
| self.putData() | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | class FitsReader(ProcessingUnit): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | # __TIMEZONE = time.timezone | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | expName = None | |
| datetimestr = None | |||
| utc = None | |||
| nChannels = None | |||
| nSamples = None | |||
| dataBlocksPerFile = None | |||
|
José Chávez
|
r897 | comments = None | |
|
Daniel Valdez
|
r487 | lastUTTime = None | |
| header_dict = None | |||
| data = None | |||
| data_header_dict = None | |||
|
José Chávez
|
r897 | ||
| def __init__(self, **kwargs): | |||
| ProcessingUnit.__init__(self, **kwargs) | |||
|
Daniel Valdez
|
r487 | self.isConfig = False | |
| self.ext = '.fits' | |||
| self.setFile = 0 | |||
| self.flagNoMoreFiles = 0 | |||
| self.flagIsNewFile = 1 | |||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = None | |
|
Daniel Valdez
|
r487 | self.fileIndex = None | |
| self.filename = None | |||
| self.fileSize = None | |||
| self.fitsObj = None | |||
| self.timeZone = None | |||
| self.nReadBlocks = 0 | |||
| self.nTotalBlocks = 0 | |||
| self.dataOut = self.createObjByDefault() | |||
| self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup() | |||
|
José Chávez
|
r897 | self.blockIndex = 1 | |
|
Daniel Valdez
|
r487 | def createObjByDefault(self): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | dataObj = Fits() | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return dataObj | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False): | |
| try: | |||
| fitsObj = pyfits.open(filename,'readonly') | |||
| except: | |||
|
George Yong
|
r1167 | print("File %s can't be opened" %(filename)) | |
|
Miguel Valdez
|
r684 | return None | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | header = fitsObj[0].header | |
| struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S") | |||
| utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | ltc = utc | |
| if useLocalTime: | |||
| ltc -= time.timezone | |||
| thisDatetime = datetime.datetime.utcfromtimestamp(ltc) | |||
| thisTime = thisDatetime.time() | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not ((startTime <= thisTime) and (endTime > thisTime)): | |
| return None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return thisDatetime | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def __setNextFileOnline(self): | |
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def __setNextFileOffline(self): | |
| idFile = self.fileIndex | |||
| while (True): | |||
| idFile += 1 | |||
| if not(idFile < len(self.filenameList)): | |||
| self.flagNoMoreFiles = 1 | |||
|
George Yong
|
r1167 | print("No more Files") | |
|
Daniel Valdez
|
r487 | return 0 | |
| filename = self.filenameList[idFile] | |||
| # if not(self.__verifyFile(filename)): | |||
| # continue | |||
| fileSize = os.path.getsize(filename) | |||
| fitsObj = pyfits.open(filename,'readonly') | |||
| break | |||
| self.flagIsNewFile = 1 | |||
| self.fileIndex = idFile | |||
| self.filename = filename | |||
| self.fileSize = fileSize | |||
| self.fitsObj = fitsObj | |||
| self.blockIndex = 0 | |||
|
George Yong
|
r1167 | print("Setting the file: %s"%self.filename) | |
|
Daniel Valdez
|
r487 | ||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r587 | def __setValuesFromHeader(self): | |
|
José Chávez
|
r897 | ||
| self.dataOut.header = self.header_dict | |||
|
Miguel Valdez
|
r587 | self.dataOut.expName = self.expName | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r587 | self.dataOut.timeZone = self.timeZone | |
| self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile | |||
|
José Chávez
|
r897 | self.dataOut.comments = self.comments | |
|
Miguel Valdez
|
r587 | # self.dataOut.timeInterval = self.timeInterval | |
| self.dataOut.channelList = self.channelList | |||
| self.dataOut.heightList = self.heightList | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r587 | self.dataOut.nCohInt = self.nCohInt | |
| self.dataOut.nIncohInt = self.nIncohInt | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | self.dataOut.ippSeconds = self.ippSeconds | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def readHeader(self): | |
| headerObj = self.fitsObj[0] | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.header_dict = headerObj.header | |
|
George Yong
|
r1167 | if 'EXPNAME' in list(headerObj.header.keys()): | |
|
Daniel Valdez
|
r487 | self.expName = headerObj.header['EXPNAME'] | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | if 'DATATYPE' in list(headerObj.header.keys()): | |
|
Daniel Valdez
|
r487 | self.dataType = headerObj.header['DATATYPE'] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.datetimestr = headerObj.header['DATETIME'] | |
| channelList = headerObj.header['CHANNELLIST'] | |||
| channelList = channelList.split('[') | |||
| channelList = channelList[1].split(']') | |||
| channelList = channelList[0].split(',') | |||
| channelList = [int(ch) for ch in channelList] | |||
| self.channelList = channelList | |||
| self.nChannels = headerObj.header['NCHANNELS'] | |||
| self.nHeights = headerObj.header['NHEIGHTS'] | |||
| self.ippSeconds = headerObj.header['IPPSECONDS'] | |||
| self.nCohInt = headerObj.header['NCOHINT'] | |||
| self.nIncohInt = headerObj.header['NINCOHINT'] | |||
| self.dataBlocksPerFile = headerObj.header['NBLOCK'] | |||
| self.timeZone = headerObj.header['TIMEZONE'] | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r527 | # self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | if 'COMMENT' in list(headerObj.header.keys()): | |
|
Daniel Valdez
|
r487 | self.comments = headerObj.header['COMMENT'] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.readHeightList() | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def readHeightList(self): | |
| self.blockIndex = self.blockIndex + 1 | |||
| obj = self.fitsObj[self.blockIndex] | |||
| self.heightList = obj.data | |||
| self.blockIndex = self.blockIndex + 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def readExtension(self): | |
| obj = self.fitsObj[self.blockIndex] | |||
| self.heightList = obj.data | |||
| self.blockIndex = self.blockIndex + 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def setNextFile(self): | |
| if self.online: | |||
| newFile = self.__setNextFileOnline() | |||
| else: | |||
| newFile = self.__setNextFileOffline() | |||
| if not(newFile): | |||
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.readHeader() | |
|
Miguel Valdez
|
r587 | self.__setValuesFromHeader() | |
|
Daniel Valdez
|
r487 | self.nReadBlocks = 0 | |
| # self.blockIndex = 1 | |||
| return 1 | |||
Juan C. Espinoza
|
r1052 | def searchFilesOffLine(self, | |
|
Daniel Valdez
|
r487 | path, | |
| startDate, | |||
| endDate, | |||
| startTime=datetime.time(0,0,0), | |||
| endTime=datetime.time(23,59,59), | |||
| set=None, | |||
| expLabel='', | |||
| ext='.fits', | |||
| walk=True): | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | pathList = [] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not walk: | |
| pathList.append(path) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | else: | |
| dirList = [] | |||
| for thisPath in os.listdir(path): | |||
| if not os.path.isdir(os.path.join(path,thisPath)): | |||
| continue | |||
|
Miguel Valdez
|
r589 | if not isRadarFolder(thisPath): | |
|
Daniel Valdez
|
r487 | continue | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | dirList.append(thisPath) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not(dirList): | |
| return None, None | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | thisDate = startDate | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | while(thisDate <= endDate): | |
| year = thisDate.timetuple().tm_year | |||
| doy = thisDate.timetuple().tm_yday | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*') | |
| if len(matchlist) == 0: | |||
| thisDate += datetime.timedelta(1) | |||
| continue | |||
| for match in matchlist: | |||
| pathList.append(os.path.join(path,match,expLabel)) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | thisDate += datetime.timedelta(1) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if pathList == []: | |
|
George Yong
|
r1167 | print("Any folder was found for the date range: %s-%s" %(startDate, endDate)) | |
|
Daniel Valdez
|
r487 | return None, None | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | print("%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | filenameList = [] | |
| datetimeList = [] | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for i in range(len(pathList)): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | thisPath = pathList[i] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | fileList = glob.glob1(thisPath, "*%s" %ext) | |
| fileList.sort() | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | for thisFile in fileList: | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | filename = os.path.join(thisPath,thisFile) | |
|
Daniel Valdez
|
r487 | thisDatetime = self.isFileinThisTime(filename, startTime, endTime) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not(thisDatetime): | |
| continue | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | filenameList.append(filename) | |
| datetimeList.append(thisDatetime) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not(filenameList): | |
|
George Yong
|
r1167 | print("Any file was found for the time range %s - %s" %(startTime, endTime)) | |
|
Daniel Valdez
|
r487 | return None, None | |
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | print("%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)) | |
| print() | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for i in range(len(filenameList)): | |
|
George Yong
|
r1167 | print("%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())) | |
|
Daniel Valdez
|
r487 | ||
| self.filenameList = filenameList | |||
| self.datetimeList = datetimeList | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return pathList, filenameList | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def setup(self, path=None, | |
|
José Chávez
|
r897 | startDate=None, | |
| endDate=None, | |||
| startTime=datetime.time(0,0,0), | |||
| endTime=datetime.time(23,59,59), | |||
| set=0, | |||
| expLabel = "", | |||
| ext = None, | |||
|
Daniel Valdez
|
r487 | online = False, | |
| delay = 60, | |||
| walk = True): | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if path == None: | |
|
George Yong
|
r1167 | raise ValueError("The path is not valid") | |
|
Daniel Valdez
|
r487 | ||
| if ext == None: | |||
| ext = self.ext | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not(online): | |
|
George Yong
|
r1167 | print("Searching files in offline mode ...") | |
|
Juan C. Espinoza
|
r1052 | pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate, | |
|
Daniel Valdez
|
r487 | startTime=startTime, endTime=endTime, | |
| set=set, expLabel=expLabel, ext=ext, | |||
| walk=walk) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not(pathList): | |
|
George Yong
|
r1167 | print("No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path, | |
|
Daniel Valdez
|
r487 | datetime.datetime.combine(startDate,startTime).ctime(), | |
|
George Yong
|
r1167 | datetime.datetime.combine(endDate,endTime).ctime())) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | sys.exit(-1) | |
| self.fileIndex = -1 | |||
| self.pathList = pathList | |||
| self.filenameList = filenameList | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.online = online | |
| self.delay = delay | |||
| ext = ext.lower() | |||
| self.ext = ext | |||
| if not(self.setNextFile()): | |||
| if (startDate!=None) and (endDate!=None): | |||
|
George Yong
|
r1167 | print("No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())) | |
|
Daniel Valdez
|
r487 | elif startDate != None: | |
|
George Yong
|
r1167 | print("No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())) | |
|
Daniel Valdez
|
r487 | else: | |
|
George Yong
|
r1167 | print("No files") | |
|
Daniel Valdez
|
r487 | ||
| sys.exit(-1) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def readBlock(self): | |
| dataObj = self.fitsObj[self.blockIndex] | |||
| self.data = dataObj.data | |||
| self.data_header_dict = dataObj.header | |||
| self.utc = self.data_header_dict['UTCTIME'] | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.flagIsNewFile = 0 | |
| self.blockIndex += 1 | |||
| self.nTotalBlocks += 1 | |||
| self.nReadBlocks += 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def __jumpToLastBlock(self): | |
|
Miguel Valdez
|
r684 | raise NotImplementedError | |
|
Daniel Valdez
|
r487 | ||
| def __waitNewBlock(self): | |||
| """ | |||
|
José Chávez
|
r897 | Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma. | |
|
Daniel Valdez
|
r487 | Si el modo de lectura es OffLine siempre retorn 0 | |
| """ | |||
| if not self.online: | |||
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if (self.nReadBlocks >= self.dataBlocksPerFile): | |
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | currentPointer = self.fp.tell() | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for nTries in range( self.nTries ): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.fp.close() | |
| self.fp = open( self.filename, 'rb' ) | |||
| self.fp.seek( currentPointer ) | |||
| self.fileSize = os.path.getsize( self.filename ) | |||
| currentSize = self.fileSize - currentPointer | |||
| if ( currentSize >= neededSize ): | |||
| self.__rdBasicHeader() | |||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | print("\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)) | |
|
Miguel Valdez
|
r568 | sleep( self.delay ) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 0 | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def __setNewBlock(self): | |
| if self.online: | |||
| self.__jumpToLastBlock() | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.flagIsNewFile: | |
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.lastUTTime = self.utc | |
| if self.online: | |||
| if self.__waitNewBlock(): | |||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.nReadBlocks < self.dataBlocksPerFile: | |
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not(self.setNextFile()): | |
| return 0 | |||
|
José Chávez
|
r897 | ||
| deltaTime = self.utc - self.lastUTTime | |||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = 0 | |
|
Daniel Valdez
|
r487 | ||
| if deltaTime > self.maxTimeStep: | |||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = 1 | |
|
Daniel Valdez
|
r487 | ||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def readNextBlock(self): | |
| if not(self.__setNewBlock()): | |||
| return 0 | |||
| if not(self.readBlock()): | |||
| return 0 | |||
| return 1 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | def printInfo(self): | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | pass | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def getData(self): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.flagNoMoreFiles: | |
| self.dataOut.flagNoData = True | |||
|
George Yong
|
r1167 | print('Process finished') | |
|
Daniel Valdez
|
r487 | return 0 | |
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r568 | self.flagDiscontinuousBlock = 0 | |
|
Daniel Valdez
|
r487 | self.flagIsNewBlock = 0 | |
| if not(self.readNextBlock()): | |||
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Miguel Valdez
|
r664 | if self.data is None: | |
|
Daniel Valdez
|
r487 | self.dataOut.flagNoData = True | |
| return 0 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.dataOut.data = self.data | |
| self.dataOut.data_header = self.data_header_dict | |||
| self.dataOut.utctime = self.utc | |||
|
José Chávez
|
r897 | ||
| # self.dataOut.header = self.header_dict | |||
|
Miguel Valdez
|
r587 | # self.dataOut.expName = self.expName | |
| # self.dataOut.nChannels = self.nChannels | |||
| # self.dataOut.timeZone = self.timeZone | |||
| # self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile | |||
|
José Chávez
|
r897 | # self.dataOut.comments = self.comments | |
|
Miguel Valdez
|
r587 | # # self.dataOut.timeInterval = self.timeInterval | |
| # self.dataOut.channelList = self.channelList | |||
| # self.dataOut.heightList = self.heightList | |||
|
Daniel Valdez
|
r487 | self.dataOut.flagNoData = False | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return self.dataOut.data | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def run(self, **kwargs): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not(self.isConfig): | |
| self.setup(**kwargs) | |||
| self.isConfig = True | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.getData() | |
|
José Chávez
|
r897 | class SpectraHeisWriter(Operation): | |
|
Daniel Valdez
|
r487 | # set = None | |
| setFile = None | |||
| idblock = None | |||
| doypath = None | |||
| subfolder = None | |||
|
José Chávez
|
r897 | ||
| def __init__(self, **kwargs): | |||
| Operation.__init__(self, **kwargs) | |||
|
Miguel Valdez
|
r664 | self.wrObj = PyFits() | |
|
José Chávez
|
r897 | # self.dataOut = dataOut | |
|
Daniel Valdez
|
r487 | self.nTotalBlocks=0 | |
| # self.set = None | |||
| self.setFile = None | |||
| self.idblock = 0 | |||
| self.wrpath = None | |||
| self.doypath = None | |||
| self.subfolder = None | |||
|
José Chávez
|
r897 | self.isConfig = False | |
|
Daniel Valdez
|
r487 | def isNumber(str): | |
| """ | |||
| Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero. | |||
|
José Chávez
|
r897 | Excepciones: | |
|
Daniel Valdez
|
r487 | Si un determinado string no puede ser convertido a numero | |
| Input: | |||
| str, string al cual se le analiza para determinar si convertible a un numero o no | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | Return: | |
| True : si el string es uno numerico | |||
| False : no es un string numerico | |||
| """ | |||
| try: | |||
| float( str ) | |||
| return True | |||
| except: | |||
|
José Chávez
|
r897 | return False | |
|
Daniel Valdez
|
r487 | def setup(self, dataOut, wrpath): | |
| if not(os.path.exists(wrpath)): | |||
| os.mkdir(wrpath) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.wrpath = wrpath | |
| # self.setFile = 0 | |||
| self.dataOut = dataOut | |||
| def putData(self): | |||
| name= time.localtime( self.dataOut.utctime) | |||
|
José Chávez
|
r897 | ext=".fits" | |
|
Daniel Valdez
|
r487 | if self.doypath == None: | |
| self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple())) | |||
| self.doypath = os.path.join( self.wrpath, self.subfolder ) | |||
| os.mkdir(self.doypath) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if self.setFile == None: | |
| # self.set = self.dataOut.set | |||
| self.setFile = 0 | |||
| # if self.set != self.dataOut.set: | |||
| ## self.set = self.dataOut.set | |||
| # self.setFile = 0 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | #make the filename | |
|
José Chávez
|
r897 | thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext) | |
| filename = os.path.join(self.wrpath,self.subfolder, thisFile) | |||
|
Daniel Valdez
|
r487 | ||
| idblock = numpy.array([self.idblock],dtype="int64") | |||
|
José Chávez
|
r897 | header=self.wrObj.cFImage(idblock=idblock, | |
|
Daniel Valdez
|
r487 | year=time.gmtime(self.dataOut.utctime).tm_year, | |
| month=time.gmtime(self.dataOut.utctime).tm_mon, | |||
|
José Chávez
|
r897 | day=time.gmtime(self.dataOut.utctime).tm_mday, | |
|
Daniel Valdez
|
r487 | hour=time.gmtime(self.dataOut.utctime).tm_hour, | |
|
José Chávez
|
r897 | minute=time.gmtime(self.dataOut.utctime).tm_min, | |
|
Daniel Valdez
|
r487 | second=time.gmtime(self.dataOut.utctime).tm_sec) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | c=3E8 | |
| deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0] | |||
| freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000)) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | colList = [] | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | colList.append(colFreq) | |
| nchannel=self.dataOut.nChannels | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | for i in range(nchannel): | |
| col = self.wrObj.writeData(name="PCh"+str(i+1), | |||
| format=str(self.dataOut.nFFTPoints)+'E', | |||
| data=10*numpy.log10(self.dataOut.data_spc[i,:])) | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | colList.append(col) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | data=self.wrObj.Ctable(colList=colList) | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.wrObj.CFile(header,data) | |
|
José Chávez
|
r897 | ||
| self.wrObj.wFile(filename) | |||
|
Daniel Valdez
|
r487 | #update the setFile | |
| self.setFile += 1 | |||
| self.idblock += 1 | |||
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | return 1 | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | def run(self, dataOut, **kwargs): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | if not(self.isConfig): | |
|
José Chávez
|
r897 | ||
|
Daniel Valdez
|
r487 | self.setup(dataOut, **kwargs) | |
| self.isConfig = True | |||
|
José Chávez
|
r897 | ||
|
George Yong
|
r1167 | self.putData() |