##// END OF EJS Templates
schain
RSS
Ultima version de jrIO_hf.py
Alexander Valdez -
r547:342b251d184b
parent child
Show More
Show file before | Show file after | Hide comments
@@ -15,6 +15,116 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 from model.data.jrodata import Voltage
15 from model.data.jrodata import Voltage
16 from model.proc.jroproc_base import ProcessingUnit, Operation
16 from model.proc.jroproc_base import ProcessingUnit, Operation
17
17
18
19 def isNumber(str):
20 """
21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22
23 Excepciones:
24 Si un determinado string no puede ser convertido a numero
25 Input:
26 str, string al cual se le analiza para determinar si convertible a un numero o no
27
28 Return:
29 True : si el string es uno numerico
30 False : no es un string numerico
31 """
32 try:
33 float( str )
34 return True
35 except:
36 return False
37
38 def getFileFromSet(path, ext, set):
39 validFilelist = []
40 fileList = os.listdir(path)
41
42 # 0 1234 567 89A BCDE
43 # H YYYY DDD SSS .ext
44
45 for thisFile in fileList:
46 try:
47 number= int(thisFile[4:10])
48
49 # year = int(thisFile[1:5])
50 # doy = int(thisFile[5:8])
51 except:
52 continue
53
54 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
55 continue
56
57 validFilelist.append(thisFile)
58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
59 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60
61 if len(myfile)!= 0:
62 return myfile[0]
63 else:
64 filename = '*%6.6d%s'%(number,ext.lower())
65 print 'the filename %s does not exist'%filename
66 print '...going to the last file: '
67
68 if validFilelist:
69 validFilelist = sorted( validFilelist, key=str.lower )
70 return validFilelist[-1]
71
72 return None
73
74 def getlastFileFromPath(path, ext):
75 """
76 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
77 al final de la depuracion devuelve el ultimo file de la lista que quedo.
78
79 Input:
80 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
81 ext : extension de los files contenidos en una carpeta
82
83 Return:
84 El ultimo file de una determinada carpeta, no se considera el path.
85 """
86 validFilelist = []
87 fileList = os.listdir(path)
88
89 # 0 1234 567 89A BCDE
90 # H YYYY DDD SSS .ext
91
92 for thisFile in fileList:
93
94 try:
95 number= int(thisFile[4:10])
96 except:
97 print "There is a file or folder with different format"
98 if not isNumber(number):
99 continue
100
101 # year = thisFile[1:5]
102 # if not isNumber(year):
103 # continue
104
105 # doy = thisFile[5:8]
106 # if not isNumber(doy):
107 # continue
108
109 number= int(number)
110 # year = int(year)
111 # doy = int(doy)
112
113 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
114 continue
115
116
117 validFilelist.append(thisFile)
118
119
120 if validFilelist:
121 validFilelist = sorted( validFilelist, key=str.lower )
122 return validFilelist[-1]
123
124 return None
125
126
127
18 class HFReader(ProcessingUnit):
128 class HFReader(ProcessingUnit):
19 '''
129 '''
20 classdocs
130 classdocs
@@ -27,6 +137,7 class HFReader(ProcessingUnit):
27 walk = None
137 walk = None
28 isConfig = False
138 isConfig = False
29 dataOut=None
139 dataOut=None
140 nTries = 3
30 ext = ".hdf5"
141 ext = ".hdf5"
31
142
32 def __init__(self):
143 def __init__(self):
@@ -54,12 +165,15 class HFReader(ProcessingUnit):
54
165
55 self.hfFilePointer= None
166 self.hfFilePointer= None
56
167
57 self.__filename_online = None
168 self.filename_online = None
58
169
59 self.status=True
170 self.status=True
60
171
61 self.flagNoMoreFiles= False
172 self.flagNoMoreFiles= False
62
173
174 self.__waitForNewFile = 20
175
176
63 #--------------------------------------------------
177 #--------------------------------------------------
64
178
65 self.dataOut = self.createObjByDefault()
179 self.dataOut = self.createObjByDefault()
@@ -97,10 +211,12 class HFReader(ProcessingUnit):
97 self.__nSamples=1000
211 self.__nSamples=1000
98 self.__deltaHeigth=1.5
212 self.__deltaHeigth=1.5
99 self.__sample_rate=1e5
213 self.__sample_rate=1e5
100 #self.__frequency=2.72e6
214 self.__frequency=2.72e6
101 self.__frequency=3.64e6
215 #self.__frequency=3.64e6
102 self.__online = False
216 self.__online = False
103
217
218 #print "Frequency of Operation:", self.__frequency
219
104
220
105 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
221 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
106 self.path = path
222 self.path = path
@@ -143,12 +259,15 class HFReader(ProcessingUnit):
143 dirnameList = filter(lambda x:x!=None,dirnameList)
259 dirnameList = filter(lambda x:x!=None,dirnameList)
144 dirnameList = [x.string for x in dirnameList]
260 dirnameList = [x.string for x in dirnameList]
145 if not(online):
261 if not(online):
262
146 dirnameList = [self.__selDates(x) for x in dirnameList]
263 dirnameList = [self.__selDates(x) for x in dirnameList]
147 dirnameList = filter(lambda x:x!=None,dirnameList)
264 dirnameList = filter(lambda x:x!=None,dirnameList)
265
148 if len(dirnameList)>0:
266 if len(dirnameList)>0:
149 self.status = 1
267 self.status = 1
150 self.dirnameList = dirnameList
268 self.dirnameList = dirnameList
151 self.dirnameList.sort()
269 self.dirnameList.sort()
270
152 else:
271 else:
153 self.status = 0
272 self.status = 0
154 return None
273 return None
@@ -165,6 +284,7 class HFReader(ProcessingUnit):
165 dir_hf_filename= filename
284 dir_hf_filename= filename
166 fp= h5py.File(dir_hf_filename,'r')
285 fp= h5py.File(dir_hf_filename,'r')
167 hipoc=fp['t'].value
286 hipoc=fp['t'].value
287 hipoc=hipoc+self.timezone
168 date_time=stuffr.unix2datestr(hipoc)
288 date_time=stuffr.unix2datestr(hipoc)
169 fp.close()
289 fp.close()
170 year =int(date_time[0:4])
290 year =int(date_time[0:4])
@@ -181,6 +301,8 class HFReader(ProcessingUnit):
181 return 1
301 return 1
182
302
183 def __getFilenameList(self):
303 def __getFilenameList(self):
304 #print "hola"
305 #print self.dirnameList
184 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
306 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
185 self.filenameList= dirList
307 self.filenameList= dirList
186
308
@@ -199,7 +321,15 class HFReader(ProcessingUnit):
199 self.status=0
321 self.status=0
200 return None
322 return None
201 else:
323 else:
202 self.filenameList=[self.filenameList[-2]]
324 #if self.set== None:
325 self.filenameList=[self.filenameList[-1]]
326 #else:
327 # try:
328 # filename=getFileFromSet(self.path,self.ext,self.set)
329 # self.filenameList=self.path+"/"+filename
330 # except:
331 # self.filenameList=[self.filenameList[-1]]
332
203
333
204 def __searchFilesOffline(self,
334 def __searchFilesOffline(self,
205 path,
335 path,
@@ -215,6 +345,7 class HFReader(ProcessingUnit):
215 self.__checkPath()
345 self.__checkPath()
216
346
217 self.__findDataForDates()
347 self.__findDataForDates()
348 #print self.dirnameList
218
349
219 self.__selectDataForTimes()
350 self.__selectDataForTimes()
220
351
@@ -225,15 +356,53 class HFReader(ProcessingUnit):
225
356
226 def __searchFilesOnline(self,
357 def __searchFilesOnline(self,
227 path,
358 path,
228 walk=True):
359 expLabel= "",
229 startDate= datetime.datetime.utcnow().date
360 ext=None,
230 endDate= datetime.datetime.utcnow().date()
361 startDate=None,
362 endDate=None,
363 walk=True,
364 set=None):
365
366
367
231 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
368 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
369
232 self.__checkPath()
370 self.__checkPath()
371
372 fullpath=path
373
374 print "%s folder was found: " %(fullpath )
375
376 if set == None:
377 filename =getlastFileFromPath(fullpath,ext)
378 startDate= datetime.datetime.utcnow().date
379 endDate= datetime.datetime.utcnow().date()
380
381 else:
382 filename= getFileFromSet(fullpath,ext,set)
383 startDate=None
384 endDate=None
385
386 if not (filename):
387 return None,None,None,None,None
388 print "%s file was found" %(filename)
389
390 dir_hf_filename= self.path+"/"+filename
391 fp= h5py.File(dir_hf_filename,'r')
392 hipoc=fp['t'].value
393 fp.close()
394 date_time=stuffr.unix2datestr(hipoc)
395
396 year =int(date_time[0:4])
397 month=int(date_time[5:7])
398 dom =int(date_time[8:10])
399 set= int(filename[4:10])
400 self.set=set-1
401 #self.dirnameList=[filename]
233 self.__findDataForDates(online=True)
402 self.__findDataForDates(online=True)
234 self.dirnameList= [self.dirnameList[-1]]
403 #print self.dirnameList
235 self.__selectDataForTimes(online=True)
404 self.__selectDataForTimes(online=True)
236 return
405 return fullpath,filename,year,month,dom,set
237
406
238 def __setNextFile(self,online=False):
407 def __setNextFile(self,online=False):
239 """
408 """
@@ -271,29 +440,41 class HFReader(ProcessingUnit):
271 self.hfFilePointer = hfFilePointer
440 self.hfFilePointer = hfFilePointer
272 hfFilePointer.close()
441 hfFilePointer.close()
273 self.__t0=epoc
442 self.__t0=epoc
274
275
276 print "Setting the file: %s"%self.filename
443 print "Setting the file: %s"%self.filename
277
444
278
279
280 return 1
445 return 1
281
446
282 def __setNextFileOnline(self):
447 def __setNextFileOnline(self):
283 """
448 """
284 """
449 """
450
451 self.set +=1
452 if self.set>8638:
453 print "There is no file with %s "%self.set
454 return
455
456
285 filename = self.filenameList[0]
457 filename = self.filenameList[0]
286 if self.__filename_online != None:
458 if self.filename_online != None:
287 self.__selecDataForTimes(online=True)
459 self.__selectDataForTimes(online=True)
288 filename = self.filenameList[0]
460 filename = self.filenameList[0]
289 while self.__filename_online == filename:
461 while self.filename_online == filename:
290 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
462 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
291 time.sleep(self.__waitForNewFile)
463 time.sleep(self.__waitForNewFile)
464 self.__findDataForDates(True)
292 self.__selectDataForTimes(online=True)
465 self.__selectDataForTimes(online=True)
293 filename = self.filenameList[0]
466 filename = self.filenameList[0]
467 print filename
468
469 hfFilePointer=h5py.File(filename,'r')
470 self.filename_online=filename
471 epoc=hfFilePointer['t'].value
472
473 self.hfFilePointer=hfFilePointer
474 hfFilePointer.close()
475 self.__t0=epoc
476
294
477
295 self.__filename_online=filename
296 self.hfFilePointer=h5py.File(filename,'r')
297 self.flagIsNewFile = 1
478 self.flagIsNewFile = 1
298 self.filename = filename
479 self.filename = filename
299
480
@@ -304,7 +485,8 class HFReader(ProcessingUnit):
304 if not(self.status):
485 if not(self.status):
305 return None
486 return None
306
487
307 def setup(self, path = None,
488 def setup(self,
489 path = None,
308 startDate = None,
490 startDate = None,
309 endDate = None,
491 endDate = None,
310 startTime = datetime.time(0,0,0),
492 startTime = datetime.time(0,0,0),
@@ -321,16 +503,36 class HFReader(ProcessingUnit):
321 In this method we should set all initial parameters.
503 In this method we should set all initial parameters.
322
504
323 '''
505 '''
506 if path==None:
507 raise ValueError,"The path is not valid"
508
509 if ext==None:
510 ext = self.ext
511
324 self.timezone= timezone
512 self.timezone= timezone
325 self.online= online
513 self.online= online
326 self.all=all
514 self.all=all
327 if ext==None:
515
328 ext = self.ext
329
516
330 if not(online):
517 if not(online):
518 print "Searching files in offline mode..."
519
331 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
520 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
332 else:
521 else:
333 self.__searchFilesOnline(path,walk)
522 print "Searching files in online mode..."
523
524 for nTries in range(self.nTries):
525
526 fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
527
528 if fullpath:
529 break
530 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
531 time.sleep(self.delay)
532 if not(fullpath):
533 print "There ins't valid files in %s" % path
534 return None
535
334
536
335 if not(self.filenameList):
537 if not(self.filenameList):
336 print "There is no files into the folder: %s"%(path)
538 print "There is no files into the folder: %s"%(path)
General Comments 0
You need to be logged in to leave comments. Login now