##// END OF EJS Templates
schain
RSS
Ultima version de jrIO_hf.py
Alexander Valdez -
r547:342b251d184b
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,506 +1,708
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-com0419
5 5 '''
6 6
7 7 import os,sys
8 8 import time,datetime
9 9 import h5py
10 10 import numpy
11 11 import re
12 12 import stuffr
13 13
14 14 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 15 from model.data.jrodata import Voltage
16 16 from model.proc.jroproc_base import ProcessingUnit, Operation
17 17
18
19 def isNumber(str):
20 """
21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22
23 Excepciones:
24 Si un determinado string no puede ser convertido a numero
25 Input:
26 str, string al cual se le analiza para determinar si convertible a un numero o no
27
28 Return:
29 True : si el string es uno numerico
30 False : no es un string numerico
31 """
32 try:
33 float( str )
34 return True
35 except:
36 return False
37
38 def getFileFromSet(path, ext, set):
39 validFilelist = []
40 fileList = os.listdir(path)
41
42 # 0 1234 567 89A BCDE
43 # H YYYY DDD SSS .ext
44
45 for thisFile in fileList:
46 try:
47 number= int(thisFile[4:10])
48
49 # year = int(thisFile[1:5])
50 # doy = int(thisFile[5:8])
51 except:
52 continue
53
54 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
55 continue
56
57 validFilelist.append(thisFile)
58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
59 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60
61 if len(myfile)!= 0:
62 return myfile[0]
63 else:
64 filename = '*%6.6d%s'%(number,ext.lower())
65 print 'the filename %s does not exist'%filename
66 print '...going to the last file: '
67
68 if validFilelist:
69 validFilelist = sorted( validFilelist, key=str.lower )
70 return validFilelist[-1]
71
72 return None
73
74 def getlastFileFromPath(path, ext):
75 """
76 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
77 al final de la depuracion devuelve el ultimo file de la lista que quedo.
78
79 Input:
80 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
81 ext : extension de los files contenidos en una carpeta
82
83 Return:
84 El ultimo file de una determinada carpeta, no se considera el path.
85 """
86 validFilelist = []
87 fileList = os.listdir(path)
88
89 # 0 1234 567 89A BCDE
90 # H YYYY DDD SSS .ext
91
92 for thisFile in fileList:
93
94 try:
95 number= int(thisFile[4:10])
96 except:
97 print "There is a file or folder with different format"
98 if not isNumber(number):
99 continue
100
101 # year = thisFile[1:5]
102 # if not isNumber(year):
103 # continue
104
105 # doy = thisFile[5:8]
106 # if not isNumber(doy):
107 # continue
108
109 number= int(number)
110 # year = int(year)
111 # doy = int(doy)
112
113 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
114 continue
115
116
117 validFilelist.append(thisFile)
118
119
120 if validFilelist:
121 validFilelist = sorted( validFilelist, key=str.lower )
122 return validFilelist[-1]
123
124 return None
125
126
127
18 128 class HFReader(ProcessingUnit):
19 129 '''
20 130 classdocs
21 131 '''
22 132 path = None
23 133 startDate= None
24 134 endDate = None
25 135 startTime= None
26 136 endTime = None
27 137 walk = None
28 138 isConfig = False
29 139 dataOut=None
140 nTries = 3
30 141 ext = ".hdf5"
31 142
32 143 def __init__(self):
33 144 '''
34 145 Constructor
35 146 '''
36 147 ProcessingUnit.__init__(self)
37 148
38 149 self.isConfig =False
39 150
40 151 self.datablock = None
41 152
42 153 self.utc = 0
43 154
44 155 self.ext='.hdf5'
45 156
46 157 self.flagIsNewFile = 1
47 158
48 159 #-------------------------------------------------
49 160 self.fileIndex=None
50 161
51 162 self.profileIndex_offset=None
52 163
53 164 self.filenameList=[]
54 165
55 166 self.hfFilePointer= None
56 167
57 self.__filename_online = None
168 self.filename_online = None
58 169
59 170 self.status=True
60 171
61 172 self.flagNoMoreFiles= False
62 173
174 self.__waitForNewFile = 20
175
176
63 177 #--------------------------------------------------
64 178
65 179 self.dataOut = self.createObjByDefault()
66 180
67 181
68 182 def createObjByDefault(self):
69 183
70 184 dataObj = Voltage()
71 185
72 186 return dataObj
73 187
74 188 def setObjProperties(self):
75 189
76 190 pass
77 191
78 192 def getBlockDimension(self):
79 193 """
80 194 Obtiene la cantidad de puntos a leer por cada bloque de datos
81 195
82 196 Affected:
83 197 self.blocksize
84 198
85 199 Return:
86 200 None
87 201 """
88 202 pts2read =self.nChannels*self.nHeights*self.nProfiles
89 203 self.blocksize = pts2read
90 204
91 205 def __readHeader(self):
92 206
93 207 self.nProfiles = 100
94 208 self.nHeights = 1000
95 209 self.nChannels = 2
96 210 self.__firstHeigth=0
97 211 self.__nSamples=1000
98 212 self.__deltaHeigth=1.5
99 213 self.__sample_rate=1e5
100 #self.__frequency=2.72e6
101 self.__frequency=3.64e6
214 self.__frequency=2.72e6
215 #self.__frequency=3.64e6
102 216 self.__online = False
103 217
218 #print "Frequency of Operation:", self.__frequency
219
104 220
105 221 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
106 222 self.path = path
107 223 self.startDate = startDate
108 224 self.endDate = endDate
109 225 self.startTime = startTime
110 226 self.endTime = endTime
111 227 self.walk = walk
112 228
113 229 def __checkPath(self):
114 230 if os.path.exists(self.path):
115 231 self.status=1
116 232 else:
117 233 self.status=0
118 234 print 'Path %s does not exits'%self.path
119 235 return
120 236
121 237 def __selDates(self, hf_dirname_format):
122 238 try:
123 239 dir_hf_filename= self.path+"/"+hf_dirname_format
124 240 fp= h5py.File(dir_hf_filename,'r')
125 241 hipoc=fp['t'].value
126 242 fp.close()
127 243 date_time=stuffr.unix2datestr(hipoc)
128 244 year =int(date_time[0:4])
129 245 month=int(date_time[5:7])
130 246 dom =int(date_time[8:10])
131 247 thisDate= datetime.date(year,month,dom)
132 248 if (thisDate>=self.startDate and thisDate <= self.endDate):
133 249 return hf_dirname_format
134 250 except:
135 251 return None
136 252
137 253 def __findDataForDates(self,online=False):
138 254 if not(self.status):
139 255 return None
140 256
141 257 pat = '\d+.\d+'
142 258 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
143 259 dirnameList = filter(lambda x:x!=None,dirnameList)
144 260 dirnameList = [x.string for x in dirnameList]
145 261 if not(online):
262
146 263 dirnameList = [self.__selDates(x) for x in dirnameList]
147 264 dirnameList = filter(lambda x:x!=None,dirnameList)
265
148 266 if len(dirnameList)>0:
149 267 self.status = 1
150 268 self.dirnameList = dirnameList
151 269 self.dirnameList.sort()
270
152 271 else:
153 272 self.status = 0
154 273 return None
155 274
156 275 def __getTimeFromData(self):
157 276 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
158 277 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
159 278 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
160 279 print '........................................'
161 280 filter_filenameList=[]
162 281 self.filenameList.sort()
163 282 for i in range(len(self.filenameList)-1):
164 283 filename=self.filenameList[i]
165 284 dir_hf_filename= filename
166 285 fp= h5py.File(dir_hf_filename,'r')
167 286 hipoc=fp['t'].value
287 hipoc=hipoc+self.timezone
168 288 date_time=stuffr.unix2datestr(hipoc)
169 289 fp.close()
170 290 year =int(date_time[0:4])
171 291 month=int(date_time[5:7])
172 292 dom =int(date_time[8:10])
173 293 hour =int(date_time[11:13])
174 294 min =int(date_time[14:16])
175 295 sec =int(date_time[17:19])
176 296 this_time=datetime.datetime(year,month,dom,hour,min,sec)
177 297 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
178 298 filter_filenameList.append(filename)
179 299 filter_filenameList.sort()
180 300 self.filenameList = filter_filenameList
181 301 return 1
182 302
183 303 def __getFilenameList(self):
304 #print "hola"
305 #print self.dirnameList
184 306 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
185 307 self.filenameList= dirList
186 308
187 309 def __selectDataForTimes(self, online=False):
188 310
189 311 if not(self.status):
190 312 return None
191 313 self.__getFilenameList()
192 314 if not(online):
193 315 if not(self.all):
194 316 self.__getTimeFromData()
195 317 if len(self.filenameList)>0:
196 318 self.status=1
197 319 self.filenameList.sort()
198 320 else:
199 321 self.status=0
200 322 return None
201 323 else:
202 self.filenameList=[self.filenameList[-2]]
324 #if self.set== None:
325 self.filenameList=[self.filenameList[-1]]
326 #else:
327 # try:
328 # filename=getFileFromSet(self.path,self.ext,self.set)
329 # self.filenameList=self.path+"/"+filename
330 # except:
331 # self.filenameList=[self.filenameList[-1]]
332
203 333
204 334 def __searchFilesOffline(self,
205 335 path,
206 336 startDate,
207 337 endDate,
208 338 ext,
209 339 startTime=datetime.time(0,0,0),
210 340 endTime=datetime.time(23,59,59),
211 341 walk=True):
212 342
213 343 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
214 344
215 345 self.__checkPath()
216 346
217 347 self.__findDataForDates()
348 #print self.dirnameList
218 349
219 350 self.__selectDataForTimes()
220 351
221 352 for i in range(len(self.filenameList)):
222 353 print "%s"% (self.filenameList[i])
223 354
224 355 return
225 356
226 357 def __searchFilesOnline(self,
227 358 path,
228 walk=True):
229 startDate= datetime.datetime.utcnow().date
230 endDate= datetime.datetime.utcnow().date()
359 expLabel= "",
360 ext=None,
361 startDate=None,
362 endDate=None,
363 walk=True,
364 set=None):
365
366
367
231 368 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
369
232 370 self.__checkPath()
371
372 fullpath=path
373
374 print "%s folder was found: " %(fullpath )
375
376 if set == None:
377 filename =getlastFileFromPath(fullpath,ext)
378 startDate= datetime.datetime.utcnow().date
379 endDate= datetime.datetime.utcnow().date()
380
381 else:
382 filename= getFileFromSet(fullpath,ext,set)
383 startDate=None
384 endDate=None
385
386 if not (filename):
387 return None,None,None,None,None
388 print "%s file was found" %(filename)
389
390 dir_hf_filename= self.path+"/"+filename
391 fp= h5py.File(dir_hf_filename,'r')
392 hipoc=fp['t'].value
393 fp.close()
394 date_time=stuffr.unix2datestr(hipoc)
395
396 year =int(date_time[0:4])
397 month=int(date_time[5:7])
398 dom =int(date_time[8:10])
399 set= int(filename[4:10])
400 self.set=set-1
401 #self.dirnameList=[filename]
233 402 self.__findDataForDates(online=True)
234 self.dirnameList= [self.dirnameList[-1]]
403 #print self.dirnameList
235 404 self.__selectDataForTimes(online=True)
236 return
405 return fullpath,filename,year,month,dom,set
237 406
238 407 def __setNextFile(self,online=False):
239 408 """
240 409 """
241 410 if not(online):
242 411 newFile = self.__setNextFileOffline()
243 412 else:
244 413 newFile = self.__setNextFileOnline()
245 414
246 415 if not(newFile):
247 416 return 0
248 417 return 1
249 418
250 419 def __setNextFileOffline(self):
251 420 """
252 421 """
253 422 idFile= self.fileIndex
254 423 while(True):
255 424 idFile += 1
256 425 if not (idFile < len(self.filenameList)):
257 426 self.flagNoMoreFiles = 1
258 427 print "No more Files"
259 428 return 0
260 429 filename = self.filenameList[idFile]
261 430 hfFilePointer =h5py.File(filename,'r')
262 431
263 432 epoc=hfFilePointer['t'].value
264 433 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
265 434 break
266 435
267 436 self.flagIsNewFile = 1
268 437 self.fileIndex = idFile
269 438 self.filename = filename
270 439
271 440 self.hfFilePointer = hfFilePointer
272 441 hfFilePointer.close()
273 442 self.__t0=epoc
274
275
276 443 print "Setting the file: %s"%self.filename
277 444
278
279
280 445 return 1
281 446
282 447 def __setNextFileOnline(self):
283 448 """
284 449 """
450
451 self.set +=1
452 if self.set>8638:
453 print "There is no file with %s "%self.set
454 return
455
456
285 457 filename = self.filenameList[0]
286 if self.__filename_online != None:
287 self.__selecDataForTimes(online=True)
458 if self.filename_online != None:
459 self.__selectDataForTimes(online=True)
288 460 filename = self.filenameList[0]
289 while self.__filename_online == filename:
461 while self.filename_online == filename:
290 462 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
291 463 time.sleep(self.__waitForNewFile)
464 self.__findDataForDates(True)
292 465 self.__selectDataForTimes(online=True)
293 466 filename = self.filenameList[0]
467 print filename
468
469 hfFilePointer=h5py.File(filename,'r')
470 self.filename_online=filename
471 epoc=hfFilePointer['t'].value
472
473 self.hfFilePointer=hfFilePointer
474 hfFilePointer.close()
475 self.__t0=epoc
476
294 477
295 self.__filename_online=filename
296 self.hfFilePointer=h5py.File(filename,'r')
297 478 self.flagIsNewFile = 1
298 479 self.filename = filename
299 480
300 481 print "Setting the file: %s"%self.filename
301 482 return 1
302 483
303 484 def __getExpParameters(self):
304 485 if not(self.status):
305 486 return None
306 487
307 def setup(self, path = None,
488 def setup(self,
489 path = None,
308 490 startDate = None,
309 491 endDate = None,
310 492 startTime = datetime.time(0,0,0),
311 493 endTime = datetime.time(23,59,59),
312 494 set = None,
313 495 expLabel = "",
314 496 ext = None,
315 497 all=0,
316 498 timezone=0,
317 499 online = False,
318 500 delay = 60,
319 501 walk = True):
320 502 '''
321 503 In this method we should set all initial parameters.
322 504
323 505 '''
506 if path==None:
507 raise ValueError,"The path is not valid"
508
509 if ext==None:
510 ext = self.ext
511
324 512 self.timezone= timezone
325 513 self.online= online
326 514 self.all=all
327 if ext==None:
328 ext = self.ext
515
329 516
330 517 if not(online):
518 print "Searching files in offline mode..."
519
331 520 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
332 521 else:
333 self.__searchFilesOnline(path,walk)
522 print "Searching files in online mode..."
523
524 for nTries in range(self.nTries):
525
526 fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
527
528 if fullpath:
529 break
530 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
531 time.sleep(self.delay)
532 if not(fullpath):
533 print "There ins't valid files in %s" % path
534 return None
535
334 536
335 537 if not(self.filenameList):
336 538 print "There is no files into the folder: %s"%(path)
337 539 sys.exit(-1)
338 540
339 541 self.__getExpParameters()
340 542
341 543 self.fileIndex = -1
342 544
343 545 self.__setNextFile(online)
344 546
345 547 self.__readMetadata()
346 548
347 549 self.__setLocalVariables()
348 550
349 551 self.__setHeaderDO()
350 552 #self.profileIndex_offset= 0
351 553
352 554 #self.profileIndex = self.profileIndex_offset
353 555
354 556 self.isConfig = True
355 557
356 558 def __readMetadata(self):
357 559 self.__readHeader()
358 560
359 561
360 562 def __setLocalVariables(self):
361 563
362 564 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
363 565 #
364 566
365 567
366 568
367 569 self.profileIndex = 9999
368 570
369 571
370 572 def __setHeaderDO(self):
371 573
372 574
373 575 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
374 576
375 577 self.dataOut.systemHeaderObj = SystemHeader()
376 578
377 579 self.dataOut.type = "Voltage"
378 580
379 581 self.dataOut.data = None
380 582
381 583 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
382 584
383 585 self.dataOut.nProfiles = 1
384 586
385 587 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
386 588
387 589 self.dataOut.channelList = range(self.nChannels)
388 590
389 591 #self.dataOut.channelIndexList = None
390 592
391 593 self.dataOut.flagNoData = True
392 594
393 595 #Set to TRUE if the data is discontinuous
394 596 self.dataOut.flagDiscontinuousBlock = False
395 597
396 598 self.dataOut.utctime = None
397 599
398 600 self.dataOut.timeZone = 0
399 601
400 602 self.dataOut.dstFlag = 0
401 603
402 604 self.dataOut.errorCount = 0
403 605
404 606 self.dataOut.nCohInt = 1
405 607
406 608 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
407 609
408 610 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
409 611
410 612 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
411 613
412 614 self.dataOut.flagShiftFFT = False
413 615
414 616 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
415 617
416 618 #Time interval between profiles
417 619 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
418 620
419 621
420 622 self.dataOut.frequency = self.__frequency
421 623
422 624 self.dataOut.realtime = self.__online
423 625
424 626 def __hasNotDataInBuffer(self):
425 627
426 628 if self.profileIndex >= self.nProfiles:
427 629 return 1
428 630
429 631 return 0
430 632
431 633 def readNextBlock(self):
432 634 if not(self.__setNewBlock()):
433 635 return 0
434 636
435 637 if not(self.readBlock()):
436 638 return 0
437 639
438 640 return 1
439 641
440 642 def __setNewBlock(self):
441 643
442 644 if self.hfFilePointer==None:
443 645 return 0
444 646
445 647 if self.flagIsNewFile:
446 648 return 1
447 649
448 650 if self.profileIndex < self.nProfiles:
449 651 return 1
450 652
451 653 self.__setNextFile(self.online)
452 654
453 655 return 1
454 656
455 657
456 658
457 659 def readBlock(self):
458 660 fp=h5py.File(self.filename,'r')
459 661 #Puntero que apunta al archivo hdf5
460 662 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
461 663 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
462 664 fp.close()
463 665 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
464 666 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
465 667 self.datablock = numpy.array([ch0,ch1])
466 668 self.flagIsNewFile=0
467 669
468 670 self.profileIndex=0
469 671
470 672 return 1
471 673
472 674 def getData(self):
473 675 if self.flagNoMoreFiles:
474 676 self.dataOut.flagNoData = True
475 677 print 'Process finished'
476 678 return 0
477 679
478 680 if self.__hasNotDataInBuffer():
479 681 if not(self.readNextBlock()):
480 682 self.dataOut.flagNodata=True
481 683 return 0
482 684
483 685 ##############################
484 686 ##############################
485 687 self.dataOut.data = self.datablock[:,:,self.profileIndex]
486 688 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex+self.timezone
487 689 self.dataOut.profileIndex= self.profileIndex
488 690 self.dataOut.flagNoData=False
489 691 self.profileIndex +=1
490 692
491 693 return self.dataOut.data
492 694
493 695
494 696 def run(self, **kwargs):
495 697 '''
496 698 This method will be called many times so here you should put all your code
497 699 '''
498 700
499 701 if not self.isConfig:
500 702 self.setup(**kwargs)
501 703 self.isConfig = True
502 704 self.getData()
503 705
504 706
505 707
506 708 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now