##// END OF EJS Templates
schain-jc
RSS
Ultima version de jrIO_hf.py
Alexander Valdez -
r547:342b251d184b
parent child
Show More
Show file before | Show file after | Hide comments
@@ -14,6 +14,116 import stuffr
14 14 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 15 from model.data.jrodata import Voltage
16 16 from model.proc.jroproc_base import ProcessingUnit, Operation
17
18
19 def isNumber(str):
20 """
21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22
23 Excepciones:
24 Si un determinado string no puede ser convertido a numero
25 Input:
26 str, string al cual se le analiza para determinar si convertible a un numero o no
27
28 Return:
29 True : si el string es uno numerico
30 False : no es un string numerico
31 """
32 try:
33 float( str )
34 return True
35 except:
36 return False
37
38 def getFileFromSet(path, ext, set):
39 validFilelist = []
40 fileList = os.listdir(path)
41
42 # 0 1234 567 89A BCDE
43 # H YYYY DDD SSS .ext
44
45 for thisFile in fileList:
46 try:
47 number= int(thisFile[4:10])
48
49 # year = int(thisFile[1:5])
50 # doy = int(thisFile[5:8])
51 except:
52 continue
53
54 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
55 continue
56
57 validFilelist.append(thisFile)
58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
59 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60
61 if len(myfile)!= 0:
62 return myfile[0]
63 else:
64 filename = '*%6.6d%s'%(number,ext.lower())
65 print 'the filename %s does not exist'%filename
66 print '...going to the last file: '
67
68 if validFilelist:
69 validFilelist = sorted( validFilelist, key=str.lower )
70 return validFilelist[-1]
71
72 return None
73
74 def getlastFileFromPath(path, ext):
75 """
76 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
77 al final de la depuracion devuelve el ultimo file de la lista que quedo.
78
79 Input:
80 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
81 ext : extension de los files contenidos en una carpeta
82
83 Return:
84 El ultimo file de una determinada carpeta, no se considera el path.
85 """
86 validFilelist = []
87 fileList = os.listdir(path)
88
89 # 0 1234 567 89A BCDE
90 # H YYYY DDD SSS .ext
91
92 for thisFile in fileList:
93
94 try:
95 number= int(thisFile[4:10])
96 except:
97 print "There is a file or folder with different format"
98 if not isNumber(number):
99 continue
100
101 # year = thisFile[1:5]
102 # if not isNumber(year):
103 # continue
104
105 # doy = thisFile[5:8]
106 # if not isNumber(doy):
107 # continue
108
109 number= int(number)
110 # year = int(year)
111 # doy = int(doy)
112
113 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
114 continue
115
116
117 validFilelist.append(thisFile)
118
119
120 if validFilelist:
121 validFilelist = sorted( validFilelist, key=str.lower )
122 return validFilelist[-1]
123
124 return None
125
126
17 127
18 128 class HFReader(ProcessingUnit):
19 129 '''
@@ -27,6 +137,7 class HFReader(ProcessingUnit):
27 137 walk = None
28 138 isConfig = False
29 139 dataOut=None
140 nTries = 3
30 141 ext = ".hdf5"
31 142
32 143 def __init__(self):
@@ -54,11 +165,14 class HFReader(ProcessingUnit):
54 165
55 166 self.hfFilePointer= None
56 167
57 self.__filename_online = None
168 self.filename_online = None
58 169
59 170 self.status=True
60 171
61 172 self.flagNoMoreFiles= False
173
174 self.__waitForNewFile = 20
175
62 176
63 177 #--------------------------------------------------
64 178
@@ -97,10 +211,12 class HFReader(ProcessingUnit):
97 211 self.__nSamples=1000
98 212 self.__deltaHeigth=1.5
99 213 self.__sample_rate=1e5
100 #self.__frequency=2.72e6
101 self.__frequency=3.64e6
214 self.__frequency=2.72e6
215 #self.__frequency=3.64e6
102 216 self.__online = False
103 217
218 #print "Frequency of Operation:", self.__frequency
219
104 220
105 221 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
106 222 self.path = path
@@ -137,18 +253,21 class HFReader(ProcessingUnit):
137 253 def __findDataForDates(self,online=False):
138 254 if not(self.status):
139 255 return None
140
256
141 257 pat = '\d+.\d+'
142 258 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
143 259 dirnameList = filter(lambda x:x!=None,dirnameList)
144 260 dirnameList = [x.string for x in dirnameList]
145 if not(online):
261 if not(online):
262
146 263 dirnameList = [self.__selDates(x) for x in dirnameList]
147 264 dirnameList = filter(lambda x:x!=None,dirnameList)
265
148 266 if len(dirnameList)>0:
149 self.status = 1
150 self.dirnameList = dirnameList
151 self.dirnameList.sort()
267 self.status = 1
268 self.dirnameList = dirnameList
269 self.dirnameList.sort()
270
152 271 else:
153 272 self.status = 0
154 273 return None
@@ -165,6 +284,7 class HFReader(ProcessingUnit):
165 284 dir_hf_filename= filename
166 285 fp= h5py.File(dir_hf_filename,'r')
167 286 hipoc=fp['t'].value
287 hipoc=hipoc+self.timezone
168 288 date_time=stuffr.unix2datestr(hipoc)
169 289 fp.close()
170 290 year =int(date_time[0:4])
@@ -173,7 +293,7 class HFReader(ProcessingUnit):
173 293 hour =int(date_time[11:13])
174 294 min =int(date_time[14:16])
175 295 sec =int(date_time[17:19])
176 this_time=datetime.datetime(year,month,dom,hour,min,sec)
296 this_time=datetime.datetime(year,month,dom,hour,min,sec)
177 297 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
178 298 filter_filenameList.append(filename)
179 299 filter_filenameList.sort()
@@ -181,6 +301,8 class HFReader(ProcessingUnit):
181 301 return 1
182 302
183 303 def __getFilenameList(self):
304 #print "hola"
305 #print self.dirnameList
184 306 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
185 307 self.filenameList= dirList
186 308
@@ -199,7 +321,15 class HFReader(ProcessingUnit):
199 321 self.status=0
200 322 return None
201 323 else:
202 self.filenameList=[self.filenameList[-2]]
324 #if self.set== None:
325 self.filenameList=[self.filenameList[-1]]
326 #else:
327 # try:
328 # filename=getFileFromSet(self.path,self.ext,self.set)
329 # self.filenameList=self.path+"/"+filename
330 # except:
331 # self.filenameList=[self.filenameList[-1]]
332
203 333
204 334 def __searchFilesOffline(self,
205 335 path,
@@ -215,6 +345,7 class HFReader(ProcessingUnit):
215 345 self.__checkPath()
216 346
217 347 self.__findDataForDates()
348 #print self.dirnameList
218 349
219 350 self.__selectDataForTimes()
220 351
@@ -225,15 +356,53 class HFReader(ProcessingUnit):
225 356
226 357 def __searchFilesOnline(self,
227 358 path,
228 walk=True):
229 startDate= datetime.datetime.utcnow().date
230 endDate= datetime.datetime.utcnow().date()
359 expLabel= "",
360 ext=None,
361 startDate=None,
362 endDate=None,
363 walk=True,
364 set=None):
365
366
367
231 368 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
369
232 370 self.__checkPath()
371
372 fullpath=path
373
374 print "%s folder was found: " %(fullpath )
375
376 if set == None:
377 filename =getlastFileFromPath(fullpath,ext)
378 startDate= datetime.datetime.utcnow().date
379 endDate= datetime.datetime.utcnow().date()
380
381 else:
382 filename= getFileFromSet(fullpath,ext,set)
383 startDate=None
384 endDate=None
385
386 if not (filename):
387 return None,None,None,None,None
388 print "%s file was found" %(filename)
389
390 dir_hf_filename= self.path+"/"+filename
391 fp= h5py.File(dir_hf_filename,'r')
392 hipoc=fp['t'].value
393 fp.close()
394 date_time=stuffr.unix2datestr(hipoc)
395
396 year =int(date_time[0:4])
397 month=int(date_time[5:7])
398 dom =int(date_time[8:10])
399 set= int(filename[4:10])
400 self.set=set-1
401 #self.dirnameList=[filename]
233 402 self.__findDataForDates(online=True)
234 self.dirnameList= [self.dirnameList[-1]]
403 #print self.dirnameList
235 404 self.__selectDataForTimes(online=True)
236 return
405 return fullpath,filename,year,month,dom,set
237 406
238 407 def __setNextFile(self,online=False):
239 408 """
@@ -271,29 +440,41 class HFReader(ProcessingUnit):
271 440 self.hfFilePointer = hfFilePointer
272 441 hfFilePointer.close()
273 442 self.__t0=epoc
274
275
276 443 print "Setting the file: %s"%self.filename
277 444
278
279
280 445 return 1
281 446
282 447 def __setNextFileOnline(self):
283 448 """
284 449 """
450
451 self.set +=1
452 if self.set>8638:
453 print "There is no file with %s "%self.set
454 return
455
456
285 457 filename = self.filenameList[0]
286 if self.__filename_online != None:
287 self.__selecDataForTimes(online=True)
458 if self.filename_online != None:
459 self.__selectDataForTimes(online=True)
288 460 filename = self.filenameList[0]
289 while self.__filename_online == filename:
461 while self.filename_online == filename:
290 462 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
291 463 time.sleep(self.__waitForNewFile)
464 self.__findDataForDates(True)
292 465 self.__selectDataForTimes(online=True)
293 466 filename = self.filenameList[0]
294
295 self.__filename_online=filename
296 self.hfFilePointer=h5py.File(filename,'r')
467 print filename
468
469 hfFilePointer=h5py.File(filename,'r')
470 self.filename_online=filename
471 epoc=hfFilePointer['t'].value
472
473 self.hfFilePointer=hfFilePointer
474 hfFilePointer.close()
475 self.__t0=epoc
476
477
297 478 self.flagIsNewFile = 1
298 479 self.filename = filename
299 480
@@ -304,7 +485,8 class HFReader(ProcessingUnit):
304 485 if not(self.status):
305 486 return None
306 487
307 def setup(self, path = None,
488 def setup(self,
489 path = None,
308 490 startDate = None,
309 491 endDate = None,
310 492 startTime = datetime.time(0,0,0),
@@ -321,17 +503,37 class HFReader(ProcessingUnit):
321 503 In this method we should set all initial parameters.
322 504
323 505 '''
506 if path==None:
507 raise ValueError,"The path is not valid"
508
509 if ext==None:
510 ext = self.ext
511
324 512 self.timezone= timezone
325 513 self.online= online
326 514 self.all=all
327 if ext==None:
328 ext = self.ext
515
329 516
330 517 if not(online):
518 print "Searching files in offline mode..."
519
331 520 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
332 521 else:
333 self.__searchFilesOnline(path,walk)
522 print "Searching files in online mode..."
334 523
524 for nTries in range(self.nTries):
525
526 fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
527
528 if fullpath:
529 break
530 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
531 time.sleep(self.delay)
532 if not(fullpath):
533 print "There ins't valid files in %s" % path
534 return None
535
536
335 537 if not(self.filenameList):
336 538 print "There is no files into the folder: %s"%(path)
337 539 sys.exit(-1)
General Comments 0
You need to be logged in to leave comments. Login now