##// END OF EJS Templates
schain
RSS
Correccion del Modo OnLine.
Alexander Valdez -
r551:62ee7a990034
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,708 +1,710
1 '''
1 '''
2 Created on Jul 3, 2014
2 Created on Jul 3, 2014
3
3
4 @author: roj-com0419
4 @author: roj-com0419
5 '''
5 '''
6
6
7 import os,sys
7 import os,sys
8 import time,datetime
8 import time,datetime
9 import h5py
9 import h5py
10 import numpy
10 import numpy
11 import re
11 import re
12 import stuffr
12 import stuffr
13
13
14 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
14 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 from model.data.jrodata import Voltage
15 from model.data.jrodata import Voltage
16 from model.proc.jroproc_base import ProcessingUnit, Operation
16 from model.proc.jroproc_base import ProcessingUnit, Operation
17
17
18
18
19 def isNumber(str):
19 def isNumber(str):
20 """
20 """
21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22
22
23 Excepciones:
23 Excepciones:
24 Si un determinado string no puede ser convertido a numero
24 Si un determinado string no puede ser convertido a numero
25 Input:
25 Input:
26 str, string al cual se le analiza para determinar si convertible a un numero o no
26 str, string al cual se le analiza para determinar si convertible a un numero o no
27
27
28 Return:
28 Return:
29 True : si el string es uno numerico
29 True : si el string es uno numerico
30 False : no es un string numerico
30 False : no es un string numerico
31 """
31 """
32 try:
32 try:
33 float( str )
33 float( str )
34 return True
34 return True
35 except:
35 except:
36 return False
36 return False
37
37
38 def getFileFromSet(path, ext, set):
38 def getFileFromSet(path, ext, set):
39 validFilelist = []
39 validFilelist = []
40 fileList = os.listdir(path)
40 fileList = os.listdir(path)
41
41
42 # 0 1234 567 89A BCDE
42 # 0 1234 567 89A BCDE
43 # H YYYY DDD SSS .ext
43 # H YYYY DDD SSS .ext
44
44
45 for thisFile in fileList:
45 for thisFile in fileList:
46 try:
46 try:
47 number= int(thisFile[4:10])
47 number= int(thisFile[4:10])
48
48
49 # year = int(thisFile[1:5])
49 # year = int(thisFile[1:5])
50 # doy = int(thisFile[5:8])
50 # doy = int(thisFile[5:8])
51 except:
51 except:
52 continue
52 continue
53
53
54 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
54 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
55 continue
55 continue
56
56
57 validFilelist.append(thisFile)
57 validFilelist.append(thisFile)
58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
59 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
59 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60
60
61 if len(myfile)!= 0:
61 if len(myfile)!= 0:
62 return myfile[0]
62 return myfile[0]
63 else:
63 else:
64 filename = '*%6.6d%s'%(number,ext.lower())
64 filename = '*%6.6d%s'%(number,ext.lower())
65 print 'the filename %s does not exist'%filename
65 print 'the filename %s does not exist'%filename
66 print '...going to the last file: '
66 print '...going to the last file: '
67
67
68 if validFilelist:
68 if validFilelist:
69 validFilelist = sorted( validFilelist, key=str.lower )
69 validFilelist = sorted( validFilelist, key=str.lower )
70 return validFilelist[-1]
70 return validFilelist[-1]
71
71
72 return None
72 return None
73
73
74 def getlastFileFromPath(path, ext):
74 def getlastFileFromPath(path, ext):
75 """
75 """
76 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
76 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
77 al final de la depuracion devuelve el ultimo file de la lista que quedo.
77 al final de la depuracion devuelve el ultimo file de la lista que quedo.
78
78
79 Input:
79 Input:
80 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
80 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
81 ext : extension de los files contenidos en una carpeta
81 ext : extension de los files contenidos en una carpeta
82
82
83 Return:
83 Return:
84 El ultimo file de una determinada carpeta, no se considera el path.
84 El ultimo file de una determinada carpeta, no se considera el path.
85 """
85 """
86 validFilelist = []
86 validFilelist = []
87 fileList = os.listdir(path)
87 fileList = os.listdir(path)
88
88
89 # 0 1234 567 89A BCDE
89 # 0 1234 567 89A BCDE
90 # H YYYY DDD SSS .ext
90 # H YYYY DDD SSS .ext
91
91
92 for thisFile in fileList:
92 for thisFile in fileList:
93
93
94 try:
94 try:
95 number= int(thisFile[4:10])
95 number= int(thisFile[4:10])
96 except:
96 except:
97 print "There is a file or folder with different format"
97 print "There is a file or folder with different format"
98 if not isNumber(number):
98 if not isNumber(number):
99 continue
99 continue
100
100
101 # year = thisFile[1:5]
101 # year = thisFile[1:5]
102 # if not isNumber(year):
102 # if not isNumber(year):
103 # continue
103 # continue
104
104
105 # doy = thisFile[5:8]
105 # doy = thisFile[5:8]
106 # if not isNumber(doy):
106 # if not isNumber(doy):
107 # continue
107 # continue
108
108
109 number= int(number)
109 number= int(number)
110 # year = int(year)
110 # year = int(year)
111 # doy = int(doy)
111 # doy = int(doy)
112
112
113 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
113 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
114 continue
114 continue
115
115
116
116
117 validFilelist.append(thisFile)
117 validFilelist.append(thisFile)
118
118
119
119
120 if validFilelist:
120 if validFilelist:
121 validFilelist = sorted( validFilelist, key=str.lower )
121 validFilelist = sorted( validFilelist, key=str.lower )
122 return validFilelist[-1]
122 return validFilelist[-1]
123
123
124 return None
124 return None
125
125
126
126
127
127
128 class HFReader(ProcessingUnit):
128 class HFReader(ProcessingUnit):
129 '''
129 '''
130 classdocs
130 classdocs
131 '''
131 '''
132 path = None
132 path = None
133 startDate= None
133 startDate= None
134 endDate = None
134 endDate = None
135 startTime= None
135 startTime= None
136 endTime = None
136 endTime = None
137 walk = None
137 walk = None
138 isConfig = False
138 isConfig = False
139 dataOut=None
139 dataOut=None
140 nTries = 3
140 nTries = 3
141 ext = ".hdf5"
141 ext = ".hdf5"
142
142
143 def __init__(self):
143 def __init__(self):
144 '''
144 '''
145 Constructor
145 Constructor
146 '''
146 '''
147 ProcessingUnit.__init__(self)
147 ProcessingUnit.__init__(self)
148
148
149 self.isConfig =False
149 self.isConfig =False
150
150
151 self.datablock = None
151 self.datablock = None
152
152
153 self.utc = 0
153 self.utc = 0
154
154
155 self.ext='.hdf5'
155 self.ext='.hdf5'
156
156
157 self.flagIsNewFile = 1
157 self.flagIsNewFile = 1
158
158
159 #-------------------------------------------------
159 #-------------------------------------------------
160 self.fileIndex=None
160 self.fileIndex=None
161
161
162 self.profileIndex_offset=None
162 self.profileIndex_offset=None
163
163
164 self.filenameList=[]
164 self.filenameList=[]
165
165
166 self.hfFilePointer= None
166 self.hfFilePointer= None
167
167
168 self.filename_online = None
168 self.filename_online = None
169
169
170 self.status=True
170 self.status=True
171
171
172 self.flagNoMoreFiles= False
172 self.flagNoMoreFiles= False
173
173
174 self.__waitForNewFile = 20
174 self.__waitForNewFile = 3
175
175
176
176
177 #--------------------------------------------------
177 #--------------------------------------------------
178
178
179 self.dataOut = self.createObjByDefault()
179 self.dataOut = self.createObjByDefault()
180
180
181
181
182 def createObjByDefault(self):
182 def createObjByDefault(self):
183
183
184 dataObj = Voltage()
184 dataObj = Voltage()
185
185
186 return dataObj
186 return dataObj
187
187
188 def setObjProperties(self):
188 def setObjProperties(self):
189
189
190 pass
190 pass
191
191
192 def getBlockDimension(self):
192 def getBlockDimension(self):
193 """
193 """
194 Obtiene la cantidad de puntos a leer por cada bloque de datos
194 Obtiene la cantidad de puntos a leer por cada bloque de datos
195
195
196 Affected:
196 Affected:
197 self.blocksize
197 self.blocksize
198
198
199 Return:
199 Return:
200 None
200 None
201 """
201 """
202 pts2read =self.nChannels*self.nHeights*self.nProfiles
202 pts2read =self.nChannels*self.nHeights*self.nProfiles
203 self.blocksize = pts2read
203 self.blocksize = pts2read
204
204
205 def __readHeader(self):
205 def __readHeader(self):
206
206
207 self.nProfiles = 100
207 self.nProfiles = 100
208 self.nHeights = 1000
208 self.nHeights = 1000
209 self.nChannels = 2
209 self.nChannels = 2
210 self.__firstHeigth=0
210 self.__firstHeigth=0
211 self.__nSamples=1000
211 self.__nSamples=1000
212 self.__deltaHeigth=1.5
212 self.__deltaHeigth=1.5
213 self.__sample_rate=1e5
213 self.__sample_rate=1e5
214 self.__frequency=2.72e6
214 #self.__frequency=2.72e6
215 #self.__frequency=3.64e6
215 #self.__frequency=3.64e6
216 self.__frequency=None
216 self.__online = False
217 self.__online = False
217
218
218 #print "Frequency of Operation:", self.__frequency
219 #print "Frequency of Operation:", self.__frequency
219
220
220
221
221 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
222 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
222 self.path = path
223 self.path = path
223 self.startDate = startDate
224 self.startDate = startDate
224 self.endDate = endDate
225 self.endDate = endDate
225 self.startTime = startTime
226 self.startTime = startTime
226 self.endTime = endTime
227 self.endTime = endTime
227 self.walk = walk
228 self.walk = walk
228
229
229 def __checkPath(self):
230 def __checkPath(self):
230 if os.path.exists(self.path):
231 if os.path.exists(self.path):
231 self.status=1
232 self.status=1
232 else:
233 else:
233 self.status=0
234 self.status=0
234 print 'Path %s does not exits'%self.path
235 print 'Path %s does not exits'%self.path
236 return
235 return
237 return
236
238
237 def __selDates(self, hf_dirname_format):
239 def __selDates(self, hf_dirname_format):
238 try:
240 try:
239 dir_hf_filename= self.path+"/"+hf_dirname_format
241 dir_hf_filename= self.path+"/"+hf_dirname_format
240 fp= h5py.File(dir_hf_filename,'r')
242 fp= h5py.File(dir_hf_filename,'r')
241 hipoc=fp['t'].value
243 hipoc=fp['t'].value
242 fp.close()
244 fp.close()
243 date_time=stuffr.unix2datestr(hipoc)
245 date_time=stuffr.unix2datestr(hipoc)
244 year =int(date_time[0:4])
246 year =int(date_time[0:4])
245 month=int(date_time[5:7])
247 month=int(date_time[5:7])
246 dom =int(date_time[8:10])
248 dom =int(date_time[8:10])
247 thisDate= datetime.date(year,month,dom)
249 thisDate= datetime.date(year,month,dom)
248 if (thisDate>=self.startDate and thisDate <= self.endDate):
250 if (thisDate>=self.startDate and thisDate <= self.endDate):
249 return hf_dirname_format
251 return hf_dirname_format
250 except:
252 except:
251 return None
253 return None
252
254
253 def __findDataForDates(self,online=False):
255 def __findDataForDates(self,online=False):
254 if not(self.status):
256 if not(self.status):
255 return None
257 return None
256
258
257 pat = '\d+.\d+'
259 pat = '\d+.\d+'
258 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
260 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
259 dirnameList = filter(lambda x:x!=None,dirnameList)
261 dirnameList = filter(lambda x:x!=None,dirnameList)
260 dirnameList = [x.string for x in dirnameList]
262 dirnameList = [x.string for x in dirnameList]
261 if not(online):
263 if not(online):
262
264
263 dirnameList = [self.__selDates(x) for x in dirnameList]
265 dirnameList = [self.__selDates(x) for x in dirnameList]
264 dirnameList = filter(lambda x:x!=None,dirnameList)
266 dirnameList = filter(lambda x:x!=None,dirnameList)
265
267
266 if len(dirnameList)>0:
268 if len(dirnameList)>0:
267 self.status = 1
269 self.status = 1
268 self.dirnameList = dirnameList
270 self.dirnameList = dirnameList
269 self.dirnameList.sort()
271 self.dirnameList.sort()
270
272
271 else:
273 else:
272 self.status = 0
274 self.status = 0
273 return None
275 return None
274
276
275 def __getTimeFromData(self):
277 def __getTimeFromData(self):
276 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
278 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
277 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
279 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
278 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
280 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
279 print '........................................'
281 print '........................................'
280 filter_filenameList=[]
282 filter_filenameList=[]
281 self.filenameList.sort()
283 self.filenameList.sort()
282 for i in range(len(self.filenameList)-1):
284 for i in range(len(self.filenameList)-1):
283 filename=self.filenameList[i]
285 filename=self.filenameList[i]
284 dir_hf_filename= filename
286 dir_hf_filename= filename
285 fp= h5py.File(dir_hf_filename,'r')
287 fp= h5py.File(dir_hf_filename,'r')
286 hipoc=fp['t'].value
288 hipoc=fp['t'].value
287 hipoc=hipoc+self.timezone
289 hipoc=hipoc+self.timezone
288 date_time=stuffr.unix2datestr(hipoc)
290 date_time=stuffr.unix2datestr(hipoc)
289 fp.close()
291 fp.close()
290 year =int(date_time[0:4])
292 year =int(date_time[0:4])
291 month=int(date_time[5:7])
293 month=int(date_time[5:7])
292 dom =int(date_time[8:10])
294 dom =int(date_time[8:10])
293 hour =int(date_time[11:13])
295 hour =int(date_time[11:13])
294 min =int(date_time[14:16])
296 min =int(date_time[14:16])
295 sec =int(date_time[17:19])
297 sec =int(date_time[17:19])
296 this_time=datetime.datetime(year,month,dom,hour,min,sec)
298 this_time=datetime.datetime(year,month,dom,hour,min,sec)
297 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
299 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
298 filter_filenameList.append(filename)
300 filter_filenameList.append(filename)
299 filter_filenameList.sort()
301 filter_filenameList.sort()
300 self.filenameList = filter_filenameList
302 self.filenameList = filter_filenameList
301 return 1
303 return 1
302
304
303 def __getFilenameList(self):
305 def __getFilenameList(self):
304 #print "hola"
306 #print "hola"
305 #print self.dirnameList
307 #print self.dirnameList
306 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
308 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
307 self.filenameList= dirList
309 self.filenameList= dirList
308
310
309 def __selectDataForTimes(self, online=False):
311 def __selectDataForTimes(self, online=False):
310
312
311 if not(self.status):
313 if not(self.status):
312 return None
314 return None
315 #----------------
313 self.__getFilenameList()
316 self.__getFilenameList()
317 #----------------
314 if not(online):
318 if not(online):
315 if not(self.all):
319 if not(self.all):
316 self.__getTimeFromData()
320 self.__getTimeFromData()
317 if len(self.filenameList)>0:
321 if len(self.filenameList)>0:
318 self.status=1
322 self.status=1
319 self.filenameList.sort()
323 self.filenameList.sort()
320 else:
324 else:
321 self.status=0
325 self.status=0
322 return None
326 return None
323 else:
327 else:
324 #if self.set== None:
328 #if self.set== None:
325 self.filenameList=[self.filenameList[-1]]
329 self.filenameList=[self.filenameList[-1]]
326 #else:
330 #else:
327 # try:
331 # try:
328 # filename=getFileFromSet(self.path,self.ext,self.set)
332 # filename=getFileFromSet(self.path,self.ext,self.set)
329 # self.filenameList=self.path+"/"+filename
333 # self.filenameList=self.path+"/"+filename
330 # except:
334 # except:
331 # self.filenameList=[self.filenameList[-1]]
335 # self.filenameList=[self.filenameList[-1]]
332
336
333
337
334 def __searchFilesOffline(self,
338 def __searchFilesOffline(self,
335 path,
339 path,
336 startDate,
340 startDate,
337 endDate,
341 endDate,
338 ext,
342 ext,
339 startTime=datetime.time(0,0,0),
343 startTime=datetime.time(0,0,0),
340 endTime=datetime.time(23,59,59),
344 endTime=datetime.time(23,59,59),
341 walk=True):
345 walk=True):
342
346
343 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
347 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
344
348
345 self.__checkPath()
349 self.__checkPath()
346
350
347 self.__findDataForDates()
351 self.__findDataForDates()
348 #print self.dirnameList
352 #print self.dirnameList
349
353
350 self.__selectDataForTimes()
354 self.__selectDataForTimes()
351
355
352 for i in range(len(self.filenameList)):
356 for i in range(len(self.filenameList)):
353 print "%s"% (self.filenameList[i])
357 print "%s"% (self.filenameList[i])
354
358
355 return
359 return
356
360
357 def __searchFilesOnline(self,
361 def __searchFilesOnline(self,
358 path,
362 path,
359 expLabel= "",
363 expLabel= "",
360 ext=None,
364 ext=None,
361 startDate=None,
365 startDate=None,
362 endDate=None,
366 endDate=None,
363 walk=True,
367 walk=True,
364 set=None):
368 set=None):
365
369
366
370
367
371 startDate = datetime.datetime.utcnow().date()
372 endDate = datetime.datetime.utcnow().date()
373
368 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
374 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
369
375
370 self.__checkPath()
376 self.__checkPath()
371
377
372 fullpath=path
378 # fullpath=path
373
379 #
374 print "%s folder was found: " %(fullpath )
380 # print "%s folder was found: " %(fullpath )
375
381 #
376 if set == None:
382 # if set == None:
377 filename =getlastFileFromPath(fullpath,ext)
383 # filename =getlastFileFromPath(fullpath,ext)
378 startDate= datetime.datetime.utcnow().date
384 # startDate= datetime.datetime.utcnow().date
379 endDate= datetime.datetime.utcnow().date()
385 # endDate= datetime.datetime.utcnow().date()
380
386 #
381 else:
387 # else:
382 filename= getFileFromSet(fullpath,ext,set)
388 # filename= getFileFromSet(fullpath,ext,set)
383 startDate=None
389 # startDate=None
384 endDate=None
390 # endDate=None
385
391 #
386 if not (filename):
392 # if not (filename):
387 return None,None,None,None,None
393 # return None,None,None,None,None
388 print "%s file was found" %(filename)
394 # print "%s file was found" %(filename)
389
395 #
390 dir_hf_filename= self.path+"/"+filename
396 # dir_hf_filename= self.path+"/"+filename
391 fp= h5py.File(dir_hf_filename,'r')
397 # fp= h5py.File(dir_hf_filename,'r')
392 hipoc=fp['t'].value
398 # hipoc=fp['t'].value
393 fp.close()
399 # fp.close()
394 date_time=stuffr.unix2datestr(hipoc)
400 # date_time=stuffr.unix2datestr(hipoc)
395
401 #
396 year =int(date_time[0:4])
402 # year =int(date_time[0:4])
397 month=int(date_time[5:7])
403 # month=int(date_time[5:7])
398 dom =int(date_time[8:10])
404 # dom =int(date_time[8:10])
399 set= int(filename[4:10])
405 # set= int(filename[4:10])
400 self.set=set-1
406 # self.set=set-1
401 #self.dirnameList=[filename]
407 #self.dirnameList=[filename]
402 self.__findDataForDates(online=True)
408 self.__findDataForDates(online=True)
409 self.dirnameList=[self.dirnameList[-1]]
403 #print self.dirnameList
410 #print self.dirnameList
404 self.__selectDataForTimes(online=True)
411 self.__selectDataForTimes(online=True)
405 return fullpath,filename,year,month,dom,set
412 #return fullpath,filename,year,month,dom,set
413 return
406
414
407 def __setNextFile(self,online=False):
415 def __setNextFile(self,online=False):
408 """
416 """
409 """
417 """
410 if not(online):
418 if not(online):
411 newFile = self.__setNextFileOffline()
419 newFile = self.__setNextFileOffline()
412 else:
420 else:
413 newFile = self.__setNextFileOnline()
421 newFile = self.__setNextFileOnline()
414
422
415 if not(newFile):
423 if not(newFile):
416 return 0
424 return 0
417 return 1
425 return 1
418
426
419 def __setNextFileOffline(self):
427 def __setNextFileOffline(self):
420 """
428 """
421 """
429 """
422 idFile= self.fileIndex
430 idFile= self.fileIndex
423 while(True):
431 while(True):
424 idFile += 1
432 idFile += 1
425 if not (idFile < len(self.filenameList)):
433 if not (idFile < len(self.filenameList)):
426 self.flagNoMoreFiles = 1
434 self.flagNoMoreFiles = 1
427 print "No more Files"
435 print "No more Files"
428 return 0
436 return 0
429 filename = self.filenameList[idFile]
437 filename = self.filenameList[idFile]
430 hfFilePointer =h5py.File(filename,'r')
438 hfFilePointer =h5py.File(filename,'r')
431
439
432 epoc=hfFilePointer['t'].value
440 epoc=hfFilePointer['t'].value
433 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
441 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
434 break
442 break
435
443
436 self.flagIsNewFile = 1
444 self.flagIsNewFile = 1
437 self.fileIndex = idFile
445 self.fileIndex = idFile
438 self.filename = filename
446 self.filename = filename
439
447
440 self.hfFilePointer = hfFilePointer
448 self.hfFilePointer = hfFilePointer
441 hfFilePointer.close()
449 hfFilePointer.close()
442 self.__t0=epoc
450 self.__t0=epoc
443 print "Setting the file: %s"%self.filename
451 print "Setting the file: %s"%self.filename
444
452
445 return 1
453 return 1
446
454
447 def __setNextFileOnline(self):
455 def __setNextFileOnline(self):
448 """
456 """
449 """
457 """
450
451 self.set +=1
452 if self.set>8638:
453 print "There is no file with %s "%self.set
454 return
455
456
457 filename = self.filenameList[0]
458 filename = self.filenameList[0]
458 if self.filename_online != None:
459 if self.filename_online != None:
459 self.__selectDataForTimes(online=True)
460 self.__selectDataForTimes(online=True)
460 filename = self.filenameList[0]
461 filename = self.filenameList[0]
461 while self.filename_online == filename:
462 while self.filename_online == filename:
462 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
463 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
463 time.sleep(self.__waitForNewFile)
464 time.sleep(self.__waitForNewFile)
464 self.__findDataForDates(True)
465 self.__findDataForDates(True)
465 self.__selectDataForTimes(online=True)
466 self.__selectDataForTimes(online=True)
466 filename = self.filenameList[0]
467 filename = self.filenameList[0]
467 print filename
468 print filename
468
469
469 hfFilePointer=h5py.File(filename,'r')
470 hfFilePointer=h5py.File(filename,'r')
470 self.filename_online=filename
471 self.filename_online=filename
471 epoc=hfFilePointer['t'].value
472 epoc=hfFilePointer['t'].value
472
473
473 self.hfFilePointer=hfFilePointer
474 self.hfFilePointer=hfFilePointer
474 hfFilePointer.close()
475 hfFilePointer.close()
475 self.__t0=epoc
476 self.__t0=epoc
476
477
477
478
478 self.flagIsNewFile = 1
479 self.flagIsNewFile = 1
479 self.filename = filename
480 self.filename = filename
480
481
481 print "Setting the file: %s"%self.filename
482 print "Setting the file: %s"%self.filename
482 return 1
483 return 1
483
484
484 def __getExpParameters(self):
485 def __getExpParameters(self):
485 if not(self.status):
486 if not(self.status):
486 return None
487 return None
487
488
488 def setup(self,
489 def setup(self,
489 path = None,
490 path = None,
490 startDate = None,
491 startDate = None,
491 endDate = None,
492 endDate = None,
492 startTime = datetime.time(0,0,0),
493 startTime = datetime.time(0,0,0),
493 endTime = datetime.time(23,59,59),
494 endTime = datetime.time(23,59,59),
494 set = None,
495 set = None,
495 expLabel = "",
496 expLabel = "",
496 ext = None,
497 ext = None,
497 all=0,
498 all=0,
498 timezone=0,
499 timezone=0,
499 online = False,
500 online = False,
500 delay = 60,
501 delay = 60,
501 walk = True):
502 walk = True):
502 '''
503 '''
503 In this method we should set all initial parameters.
504 In this method we should set all initial parameters.
504
505
505 '''
506 '''
506 if path==None:
507 if path==None:
507 raise ValueError,"The path is not valid"
508 raise ValueError,"The path is not valid"
508
509
509 if ext==None:
510 if ext==None:
510 ext = self.ext
511 ext = self.ext
511
512
512 self.timezone= timezone
513 self.timezone= timezone
513 self.online= online
514 self.online= online
514 self.all=all
515 self.all=all
515
516
516
517
517 if not(online):
518 if not(online):
518 print "Searching files in offline mode..."
519 print "Searching files in offline mode..."
519
520
520 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
521 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
521 else:
522 else:
522 print "Searching files in online mode..."
523 print "Searching files in online mode..."
524 self.__searchFilesOnline(path, walk)
523
525
524 for nTries in range(self.nTries):
526 # for nTries in range(self.nTries):
525
527 #
526 fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
528 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
527
529 #
528 if fullpath:
530 # if fullpath:
529 break
531 # break
530 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
532 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
531 time.sleep(self.delay)
533 # time.sleep(self.delay)
532 if not(fullpath):
534 # if not(fullpath):
533 print "There ins't valid files in %s" % path
535 # print "There ins't valid files in %s" % path
534 return None
536 # return None
535
537
536
538
537 if not(self.filenameList):
539 if not(self.filenameList):
538 print "There is no files into the folder: %s"%(path)
540 print "There is no files into the folder: %s"%(path)
539 sys.exit(-1)
541 sys.exit(-1)
540
542
541 self.__getExpParameters()
543 self.__getExpParameters()
542
544
543 self.fileIndex = -1
545 self.fileIndex = -1
544
546
545 self.__setNextFile(online)
547 self.__setNextFile(online)
546
548
547 self.__readMetadata()
549 self.__readMetadata()
548
550
549 self.__setLocalVariables()
551 self.__setLocalVariables()
550
552
551 self.__setHeaderDO()
553 self.__setHeaderDO()
552 #self.profileIndex_offset= 0
554 #self.profileIndex_offset= 0
553
555
554 #self.profileIndex = self.profileIndex_offset
556 #self.profileIndex = self.profileIndex_offset
555
557
556 self.isConfig = True
558 self.isConfig = True
557
559
558 def __readMetadata(self):
560 def __readMetadata(self):
559 self.__readHeader()
561 self.__readHeader()
560
562
561
563
562 def __setLocalVariables(self):
564 def __setLocalVariables(self):
563
565
564 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
566 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
565 #
567 #
566
568
567
569
568
570
569 self.profileIndex = 9999
571 self.profileIndex = 9999
570
572
571
573
572 def __setHeaderDO(self):
574 def __setHeaderDO(self):
573
575
574
576
575 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
577 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
576
578
577 self.dataOut.systemHeaderObj = SystemHeader()
579 self.dataOut.systemHeaderObj = SystemHeader()
578
580
579 self.dataOut.type = "Voltage"
581 self.dataOut.type = "Voltage"
580
582
581 self.dataOut.data = None
583 self.dataOut.data = None
582
584
583 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
585 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
584
586
585 self.dataOut.nProfiles = 1
587 self.dataOut.nProfiles = 1
586
588
587 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
589 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
588
590
589 self.dataOut.channelList = range(self.nChannels)
591 self.dataOut.channelList = range(self.nChannels)
590
592
591 #self.dataOut.channelIndexList = None
593 #self.dataOut.channelIndexList = None
592
594
593 self.dataOut.flagNoData = True
595 self.dataOut.flagNoData = True
594
596
595 #Set to TRUE if the data is discontinuous
597 #Set to TRUE if the data is discontinuous
596 self.dataOut.flagDiscontinuousBlock = False
598 self.dataOut.flagDiscontinuousBlock = False
597
599
598 self.dataOut.utctime = None
600 self.dataOut.utctime = None
599
601
600 self.dataOut.timeZone = 0
602 self.dataOut.timeZone = 0
601
603
602 self.dataOut.dstFlag = 0
604 self.dataOut.dstFlag = 0
603
605
604 self.dataOut.errorCount = 0
606 self.dataOut.errorCount = 0
605
607
606 self.dataOut.nCohInt = 1
608 self.dataOut.nCohInt = 1
607
609
608 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
610 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
609
611
610 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
612 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
611
613
612 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
614 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
613
615
614 self.dataOut.flagShiftFFT = False
616 self.dataOut.flagShiftFFT = False
615
617
616 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
618 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
617
619
618 #Time interval between profiles
620 #Time interval between profiles
619 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
621 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
620
622
621
623
622 self.dataOut.frequency = self.__frequency
624 self.dataOut.frequency = self.__frequency
623
625
624 self.dataOut.realtime = self.__online
626 self.dataOut.realtime = self.__online
625
627
626 def __hasNotDataInBuffer(self):
628 def __hasNotDataInBuffer(self):
627
629
628 if self.profileIndex >= self.nProfiles:
630 if self.profileIndex >= self.nProfiles:
629 return 1
631 return 1
630
632
631 return 0
633 return 0
632
634
633 def readNextBlock(self):
635 def readNextBlock(self):
634 if not(self.__setNewBlock()):
636 if not(self.__setNewBlock()):
635 return 0
637 return 0
636
638
637 if not(self.readBlock()):
639 if not(self.readBlock()):
638 return 0
640 return 0
639
641
640 return 1
642 return 1
641
643
642 def __setNewBlock(self):
644 def __setNewBlock(self):
643
645
644 if self.hfFilePointer==None:
646 if self.hfFilePointer==None:
645 return 0
647 return 0
646
648
647 if self.flagIsNewFile:
649 if self.flagIsNewFile:
648 return 1
650 return 1
649
651
650 if self.profileIndex < self.nProfiles:
652 if self.profileIndex < self.nProfiles:
651 return 1
653 return 1
652
654
653 self.__setNextFile(self.online)
655 self.__setNextFile(self.online)
654
656
655 return 1
657 return 1
656
658
657
659
658
660
659 def readBlock(self):
661 def readBlock(self):
660 fp=h5py.File(self.filename,'r')
662 fp=h5py.File(self.filename,'r')
661 #Puntero que apunta al archivo hdf5
663 #Puntero que apunta al archivo hdf5
662 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
664 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
663 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
665 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
664 fp.close()
666 fp.close()
665 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
667 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
666 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
668 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
667 self.datablock = numpy.array([ch0,ch1])
669 self.datablock = numpy.array([ch0,ch1])
668 self.flagIsNewFile=0
670 self.flagIsNewFile=0
669
671
670 self.profileIndex=0
672 self.profileIndex=0
671
673
672 return 1
674 return 1
673
675
674 def getData(self):
676 def getData(self):
675 if self.flagNoMoreFiles:
677 if self.flagNoMoreFiles:
676 self.dataOut.flagNoData = True
678 self.dataOut.flagNoData = True
677 print 'Process finished'
679 print 'Process finished'
678 return 0
680 return 0
679
681
680 if self.__hasNotDataInBuffer():
682 if self.__hasNotDataInBuffer():
681 if not(self.readNextBlock()):
683 if not(self.readNextBlock()):
682 self.dataOut.flagNodata=True
684 self.dataOut.flagNodata=True
683 return 0
685 return 0
684
686
685 ##############################
687 ##############################
686 ##############################
688 ##############################
687 self.dataOut.data = self.datablock[:,:,self.profileIndex]
689 self.dataOut.data = self.datablock[:,:,self.profileIndex]
688 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex+self.timezone
690 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex+self.timezone
689 self.dataOut.profileIndex= self.profileIndex
691 self.dataOut.profileIndex= self.profileIndex
690 self.dataOut.flagNoData=False
692 self.dataOut.flagNoData=False
691 self.profileIndex +=1
693 self.profileIndex +=1
692
694
693 return self.dataOut.data
695 return self.dataOut.data
694
696
695
697
696 def run(self, **kwargs):
698 def run(self, **kwargs):
697 '''
699 '''
698 This method will be called many times so here you should put all your code
700 This method will be called many times so here you should put all your code
699 '''
701 '''
700
702
701 if not self.isConfig:
703 if not self.isConfig:
702 self.setup(**kwargs)
704 self.setup(**kwargs)
703 self.isConfig = True
705 self.isConfig = True
704 self.getData()
706 self.getData()
705
707
706
708
707
709
708 No newline at end of file
710
General Comments 0
You need to be logged in to leave comments. Login now