##// END OF EJS Templates
schain
RSS
Se agrega el metodo FitsReader y el test FitsExp.py
Daniel Valdez -
r353:f28c6f8a3e79
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,45
1 import os, sys
2
3 path = os.path.split(os.getcwd())[0]
4 sys.path.append(path)
5
6 from controller import *
7
8 desc = "FITS Test"
9 filename = "fitsexp.xml"
10
11 controllerObj = Project()
12
13 controllerObj.setup(id = '191', name='test01', description=desc)
14
15 readUnitConfObj = controllerObj.addReadUnit(datatype='Fits',
16 path='/Users/dsuarez/Remote/d2013043',
17 startDate='2013/02/06',
18 endDate='2013/12/31',
19 startTime='00:30:00',
20 endTime='17:40:59',
21 online=0,
22 delay=3,
23 walk=0)
24
25 #procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
26
27 #procUnitConfObj1 = controllerObj.addProcUnit(datatype='SpectraHeis', inputId=procUnitConfObj0.getId())
28 #procUnitConfObj1.addParameter(name='timeInterval', value='5', format='int')
29
30 #opObj11 = procUnitConfObj1.addOperation(name='IncohInt4SpectraHeis', optype='other')
31 #opObj11.addParameter(name='timeInterval', value='1', format='float')
32
33
34
35
36 print "Escribiendo el archivo XML"
37 controllerObj.writeXml(filename)
38 print "Leyendo el archivo XML"
39 controllerObj.readXml(filename)
40
41 controllerObj.createObjects()
42 controllerObj.connectObjects()
43 controllerObj.run()
44
45 No newline at end of file
Show file before | Show file after | Hide comments
@@ -572,3 +572,53 class SpectraHeis(JROData):
572 self.utctime = None
572 self.utctime = None
573
573
574 self.blocksize = None
574 self.blocksize = None
575
576 class Fits:
577
578 def __init__(self):
579 self.useLocalTime = False
580 self.utctime = None
581 self.timeZone = None
582 self.ltctime = None
583 self.timeInterval = None
584 self.header = None
585 self.data_header = None
586 self.data = None
587 self.datatime = None
588 self.flagNoData = False
589 self.expName = ''
590 self.nChannels = None
591 self.nSamples = None
592 self.dataBlocksPerFile = None
593 self.comments = ''
594
595
596 def getltctime(self):
597
598 if self.useLocalTime:
599 return self.utctime - self.timeZone*60
600
601 return self.utctime
602
603 def getDatatime(self):
604
605 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
606 return datatime
607
608 def getTimeRange(self):
609
610 datatime = []
611
612 datatime.append(self.ltctime)
613 datatime.append(self.ltctime + self.timeInterval)
614
615 datatime = numpy.array(datatime)
616
617 return datatime
618
619 def isEmpty(self):
620
621 return self.flagNoData
622
623 datatime = property(getDatatime, "I'm the 'datatime' property")
624 ltctime = property(getltctime, "I'm the 'ltctime' property") No newline at end of file
Show file before | Show file after | Hide comments
@@ -2919,3 +2919,395 class FitsWriter(Operation):
2919 self.putData()
2919 self.putData()
2920
2920
2921
2921
2922 class FitsReader(ProcessingUnit):
2923
2924 __TIMEZONE = time.timezone
2925
2926 expName = None
2927 datetimestr = None
2928 utc = None
2929 nChannels = None
2930 nSamples = None
2931 dataBlocksPerFile = None
2932 comments = None
2933 lastUTTime = None
2934 header_dict = None
2935 data = None
2936 data_header_dict = None
2937
2938 def __init__(self):
2939 self.isConfig = False
2940 self.ext = '.fits'
2941 self.setFile = 0
2942 self.flagNoMoreFiles = 0
2943 self.flagIsNewFile = 1
2944 self.flagTimeBlock = None
2945 self.fileIndex = None
2946 self.filename = None
2947 self.fileSize = None
2948 self.fitsObj = None
2949 self.nReadBlocks = 0
2950 self.nTotalBlocks = 0
2951 self.dataOut = self.createObjByDefault()
2952 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
2953 self.blockIndex = 1
2954
2955 def createObjByDefault(self):
2956
2957 dataObj = Fits()
2958
2959 return dataObj
2960
2961 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
2962 try:
2963 fitsObj = pyfits.open(filename,'readonly')
2964 except:
2965 raise IOError, "The file %s can't be opened" %(filename)
2966
2967 header = fitsObj[0].header
2968 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
2969 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
2970
2971 ltc = utc
2972 if useLocalTime:
2973 ltc -= time.timezone
2974 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
2975 thisTime = thisDatetime.time()
2976
2977 if not ((startTime <= thisTime) and (endTime > thisTime)):
2978 return None
2979
2980 return thisDatetime
2981
2982 def __setNextFileOnline(self):
2983 raise ValueError, "No implemented"
2984
2985 def __setNextFileOffline(self):
2986 idFile = self.fileIndex
2987
2988 while (True):
2989 idFile += 1
2990 if not(idFile < len(self.filenameList)):
2991 self.flagNoMoreFiles = 1
2992 print "No more Files"
2993 return 0
2994
2995 filename = self.filenameList[idFile]
2996
2997 # if not(self.__verifyFile(filename)):
2998 # continue
2999
3000 fileSize = os.path.getsize(filename)
3001 fitsObj = pyfits.open(filename,'readonly')
3002 break
3003
3004 self.flagIsNewFile = 1
3005 self.fileIndex = idFile
3006 self.filename = filename
3007 self.fileSize = fileSize
3008 self.fitsObj = fitsObj
3009
3010 print "Setting the file: %s"%self.filename
3011
3012 return 1
3013
3014 def readHeader(self):
3015 headerObj = self.fitsObj[0]
3016
3017 self.header_dict = headerObj.header
3018 self.expName = headerObj.header['EXPNAME']
3019 self.datetimestr = headerObj.header['DATETIME']
3020 struct_time = time.strptime(headerObj.header['DATETIME'], "%b %d %Y %H:%M:%S")
3021 # self.utc = time.mktime(struct_time) - self.__TIMEZONE
3022 self.nChannels = headerObj.header['NCHANNEL']
3023 self.nSamples = headerObj.header['NSAMPLE']
3024 self.dataBlocksPerFile = headerObj.header['NBLOCK']
3025 self.comments = headerObj.header['COMMENT']
3026
3027
3028 def setNextFile(self):
3029
3030 if self.online:
3031 newFile = self.__setNextFileOnline()
3032 else:
3033 newFile = self.__setNextFileOffline()
3034
3035 if not(newFile):
3036 return 0
3037
3038 self.readHeader()
3039
3040 self.nReadBlocks = 0
3041 self.blockIndex = 1
3042 return 1
3043
3044 def __searchFilesOffLine(self,
3045 path,
3046 startDate,
3047 endDate,
3048 startTime=datetime.time(0,0,0),
3049 endTime=datetime.time(23,59,59),
3050 set=None,
3051 expLabel='',
3052 ext='.fits',
3053 walk=True):
3054
3055 pathList = []
3056
3057 if not walk:
3058 pathList.append(path)
3059
3060 else:
3061 dirList = []
3062 for thisPath in os.listdir(path):
3063 if not os.path.isdir(os.path.join(path,thisPath)):
3064 continue
3065 if not isDoyFolder(thisPath):
3066 continue
3067
3068 dirList.append(thisPath)
3069
3070 if not(dirList):
3071 return None, None
3072
3073 thisDate = startDate
3074
3075 while(thisDate <= endDate):
3076 year = thisDate.timetuple().tm_year
3077 doy = thisDate.timetuple().tm_yday
3078
3079 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
3080 if len(matchlist) == 0:
3081 thisDate += datetime.timedelta(1)
3082 continue
3083 for match in matchlist:
3084 pathList.append(os.path.join(path,match,expLabel))
3085
3086 thisDate += datetime.timedelta(1)
3087
3088 if pathList == []:
3089 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
3090 return None, None
3091
3092 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
3093
3094 filenameList = []
3095 datetimeList = []
3096
3097 for i in range(len(pathList)):
3098
3099 thisPath = pathList[i]
3100
3101 fileList = glob.glob1(thisPath, "*%s" %ext)
3102 fileList.sort()
3103
3104 for file in fileList:
3105
3106 filename = os.path.join(thisPath,file)
3107 thisDatetime = self.isFileinThisTime(filename, startTime, endTime, useLocalTime=True)
3108
3109 if not(thisDatetime):
3110 continue
3111
3112 filenameList.append(filename)
3113 datetimeList.append(thisDatetime)
3114
3115 if not(filenameList):
3116 print "Any file was found for the time range %s - %s" %(startTime, endTime)
3117 return None, None
3118
3119 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
3120 print
3121
3122 for i in range(len(filenameList)):
3123 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
3124
3125 self.filenameList = filenameList
3126 self.datetimeList = datetimeList
3127
3128 return pathList, filenameList
3129
3130 def setup(self, path=None,
3131 startDate=None,
3132 endDate=None,
3133 startTime=datetime.time(0,0,0),
3134 endTime=datetime.time(23,59,59),
3135 set=0,
3136 expLabel = "",
3137 ext = None,
3138 online = False,
3139 delay = 60,
3140 walk = True):
3141
3142 if path == None:
3143 raise ValueError, "The path is not valid"
3144
3145 if ext == None:
3146 ext = self.ext
3147
3148 if not(online):
3149 print "Searching files in offline mode ..."
3150 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
3151 startTime=startTime, endTime=endTime,
3152 set=set, expLabel=expLabel, ext=ext,
3153 walk=walk)
3154
3155 if not(pathList):
3156 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
3157 datetime.datetime.combine(startDate,startTime).ctime(),
3158 datetime.datetime.combine(endDate,endTime).ctime())
3159
3160 sys.exit(-1)
3161
3162 self.fileIndex = -1
3163 self.pathList = pathList
3164 self.filenameList = filenameList
3165
3166 self.online = online
3167 self.delay = delay
3168 ext = ext.lower()
3169 self.ext = ext
3170
3171 if not(self.setNextFile()):
3172 if (startDate!=None) and (endDate!=None):
3173 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
3174 elif startDate != None:
3175 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
3176 else:
3177 print "No files"
3178
3179 sys.exit(-1)
3180
3181
3182
3183 def readBlock(self):
3184 dataObj = self.fitsObj[self.blockIndex]
3185
3186 self.data = dataObj.data
3187 self.data_header_dict = dataObj.header
3188 self.utc = self.data_header_dict['UTCTIME']
3189
3190 self.flagIsNewFile = 0
3191 self.blockIndex += 1
3192 self.nTotalBlocks += 1
3193 self.nReadBlocks += 1
3194
3195 return 1
3196
3197 def __jumpToLastBlock(self):
3198 raise ValueError, "No implemented"
3199
3200 def __waitNewBlock(self):
3201 """
3202 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
3203
3204 Si el modo de lectura es OffLine siempre retorn 0
3205 """
3206 if not self.online:
3207 return 0
3208
3209 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
3210 return 0
3211
3212 currentPointer = self.fp.tell()
3213
3214 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
3215
3216 for nTries in range( self.nTries ):
3217
3218 self.fp.close()
3219 self.fp = open( self.filename, 'rb' )
3220 self.fp.seek( currentPointer )
3221
3222 self.fileSize = os.path.getsize( self.filename )
3223 currentSize = self.fileSize - currentPointer
3224
3225 if ( currentSize >= neededSize ):
3226 self.__rdBasicHeader()
3227 return 1
3228
3229 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
3230 time.sleep( self.delay )
3231
3232
3233 return 0
3234
3235 def __setNewBlock(self):
3236
3237 if self.online:
3238 self.__jumpToLastBlock()
3239
3240 if self.flagIsNewFile:
3241 return 1
3242
3243 self.lastUTTime = self.utc
3244
3245 if self.online:
3246 if self.__waitNewBlock():
3247 return 1
3248
3249 if self.nReadBlocks < self.dataBlocksPerFile:
3250 return 1
3251
3252 if not(self.setNextFile()):
3253 return 0
3254
3255 deltaTime = self.utc - self.lastUTTime
3256
3257 self.flagTimeBlock = 0
3258
3259 if deltaTime > self.maxTimeStep:
3260 self.flagTimeBlock = 1
3261
3262 return 1
3263
3264
3265 def readNextBlock(self):
3266 if not(self.__setNewBlock()):
3267 return 0
3268
3269 if not(self.readBlock()):
3270 return 0
3271
3272 return 1
3273
3274
3275 def getData(self):
3276
3277 if self.flagNoMoreFiles:
3278 self.dataOut.flagNoData = True
3279 print 'Process finished'
3280 return 0
3281
3282 self.flagTimeBlock = 0
3283 self.flagIsNewBlock = 0
3284
3285 if not(self.readNextBlock()):
3286 return 0
3287
3288 if self.data == None:
3289 self.dataOut.flagNoData = True
3290 return 0
3291
3292 self.dataOut.data = self.data
3293 self.dataOut.data_header = self.data_header_dict
3294 self.dataOut.utctime = self.utc
3295
3296 self.dataOut.header = self.header_dict
3297 self.dataOut.expName = self.expName
3298 self.dataOut.nChannels = self.nChannels
3299 self.dataOut.nSamples = self.nSamples
3300 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
3301 self.dataOut.comments = self.comments
3302
3303 self.dataOut.flagNoData = False
3304
3305 return self.dataOut.data
3306
3307 def run(self, **kwargs):
3308
3309 if not(self.isConfig):
3310 self.setup(**kwargs)
3311 self.isConfig = True
3312
3313 self.getData() No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now