##// END OF EJS Templates
schain-jc
RSS
Correccion del Modo OnLine.
Alexander Valdez -
r551:62ee7a990034
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,708 +1,710
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-com0419
5 5 '''
6 6
7 7 import os,sys
8 8 import time,datetime
9 9 import h5py
10 10 import numpy
11 11 import re
12 12 import stuffr
13 13
14 14 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 15 from model.data.jrodata import Voltage
16 16 from model.proc.jroproc_base import ProcessingUnit, Operation
17 17
18 18
19 19 def isNumber(str):
20 20 """
21 21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22 22
23 23 Excepciones:
24 24 Si un determinado string no puede ser convertido a numero
25 25 Input:
26 26 str, string al cual se le analiza para determinar si convertible a un numero o no
27 27
28 28 Return:
29 29 True : si el string es uno numerico
30 30 False : no es un string numerico
31 31 """
32 32 try:
33 33 float( str )
34 34 return True
35 35 except:
36 36 return False
37 37
38 38 def getFileFromSet(path, ext, set):
39 39 validFilelist = []
40 40 fileList = os.listdir(path)
41 41
42 42 # 0 1234 567 89A BCDE
43 43 # H YYYY DDD SSS .ext
44 44
45 45 for thisFile in fileList:
46 46 try:
47 47 number= int(thisFile[4:10])
48 48
49 49 # year = int(thisFile[1:5])
50 50 # doy = int(thisFile[5:8])
51 51 except:
52 52 continue
53 53
54 54 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
55 55 continue
56 56
57 57 validFilelist.append(thisFile)
58 58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
59 59 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60 60
61 61 if len(myfile)!= 0:
62 62 return myfile[0]
63 63 else:
64 64 filename = '*%6.6d%s'%(number,ext.lower())
65 65 print 'the filename %s does not exist'%filename
66 66 print '...going to the last file: '
67 67
68 68 if validFilelist:
69 69 validFilelist = sorted( validFilelist, key=str.lower )
70 70 return validFilelist[-1]
71 71
72 72 return None
73 73
74 74 def getlastFileFromPath(path, ext):
75 75 """
76 76 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
77 77 al final de la depuracion devuelve el ultimo file de la lista que quedo.
78 78
79 79 Input:
80 80 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
81 81 ext : extension de los files contenidos en una carpeta
82 82
83 83 Return:
84 84 El ultimo file de una determinada carpeta, no se considera el path.
85 85 """
86 86 validFilelist = []
87 87 fileList = os.listdir(path)
88 88
89 89 # 0 1234 567 89A BCDE
90 90 # H YYYY DDD SSS .ext
91 91
92 92 for thisFile in fileList:
93 93
94 94 try:
95 95 number= int(thisFile[4:10])
96 96 except:
97 97 print "There is a file or folder with different format"
98 98 if not isNumber(number):
99 99 continue
100 100
101 101 # year = thisFile[1:5]
102 102 # if not isNumber(year):
103 103 # continue
104 104
105 105 # doy = thisFile[5:8]
106 106 # if not isNumber(doy):
107 107 # continue
108 108
109 109 number= int(number)
110 110 # year = int(year)
111 111 # doy = int(doy)
112 112
113 113 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
114 114 continue
115 115
116 116
117 117 validFilelist.append(thisFile)
118 118
119 119
120 120 if validFilelist:
121 121 validFilelist = sorted( validFilelist, key=str.lower )
122 122 return validFilelist[-1]
123 123
124 124 return None
125 125
126 126
127 127
128 128 class HFReader(ProcessingUnit):
129 129 '''
130 130 classdocs
131 131 '''
132 132 path = None
133 133 startDate= None
134 134 endDate = None
135 135 startTime= None
136 136 endTime = None
137 137 walk = None
138 138 isConfig = False
139 139 dataOut=None
140 140 nTries = 3
141 141 ext = ".hdf5"
142 142
143 143 def __init__(self):
144 144 '''
145 145 Constructor
146 146 '''
147 147 ProcessingUnit.__init__(self)
148 148
149 149 self.isConfig =False
150 150
151 151 self.datablock = None
152 152
153 153 self.utc = 0
154 154
155 155 self.ext='.hdf5'
156 156
157 157 self.flagIsNewFile = 1
158 158
159 159 #-------------------------------------------------
160 160 self.fileIndex=None
161 161
162 162 self.profileIndex_offset=None
163 163
164 164 self.filenameList=[]
165 165
166 166 self.hfFilePointer= None
167 167
168 168 self.filename_online = None
169 169
170 170 self.status=True
171 171
172 172 self.flagNoMoreFiles= False
173 173
174 self.__waitForNewFile = 20
174 self.__waitForNewFile = 3
175 175
176 176
177 177 #--------------------------------------------------
178 178
179 179 self.dataOut = self.createObjByDefault()
180 180
181 181
182 182 def createObjByDefault(self):
183 183
184 184 dataObj = Voltage()
185 185
186 186 return dataObj
187 187
188 188 def setObjProperties(self):
189 189
190 190 pass
191 191
192 192 def getBlockDimension(self):
193 193 """
194 194 Obtiene la cantidad de puntos a leer por cada bloque de datos
195 195
196 196 Affected:
197 197 self.blocksize
198 198
199 199 Return:
200 200 None
201 201 """
202 202 pts2read =self.nChannels*self.nHeights*self.nProfiles
203 203 self.blocksize = pts2read
204 204
205 205 def __readHeader(self):
206 206
207 207 self.nProfiles = 100
208 208 self.nHeights = 1000
209 209 self.nChannels = 2
210 210 self.__firstHeigth=0
211 211 self.__nSamples=1000
212 212 self.__deltaHeigth=1.5
213 213 self.__sample_rate=1e5
214 self.__frequency=2.72e6
214 #self.__frequency=2.72e6
215 215 #self.__frequency=3.64e6
216 self.__frequency=None
216 217 self.__online = False
217 218
218 219 #print "Frequency of Operation:", self.__frequency
219 220
220 221
221 222 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
222 223 self.path = path
223 224 self.startDate = startDate
224 225 self.endDate = endDate
225 226 self.startTime = startTime
226 227 self.endTime = endTime
227 228 self.walk = walk
228 229
229 230 def __checkPath(self):
230 231 if os.path.exists(self.path):
231 232 self.status=1
232 233 else:
233 234 self.status=0
234 235 print 'Path %s does not exits'%self.path
236 return
235 237 return
236 238
237 239 def __selDates(self, hf_dirname_format):
238 240 try:
239 241 dir_hf_filename= self.path+"/"+hf_dirname_format
240 242 fp= h5py.File(dir_hf_filename,'r')
241 243 hipoc=fp['t'].value
242 244 fp.close()
243 245 date_time=stuffr.unix2datestr(hipoc)
244 246 year =int(date_time[0:4])
245 247 month=int(date_time[5:7])
246 248 dom =int(date_time[8:10])
247 249 thisDate= datetime.date(year,month,dom)
248 250 if (thisDate>=self.startDate and thisDate <= self.endDate):
249 251 return hf_dirname_format
250 252 except:
251 253 return None
252 254
253 255 def __findDataForDates(self,online=False):
254 256 if not(self.status):
255 257 return None
256 258
257 259 pat = '\d+.\d+'
258 260 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
259 261 dirnameList = filter(lambda x:x!=None,dirnameList)
260 262 dirnameList = [x.string for x in dirnameList]
261 263 if not(online):
262 264
263 265 dirnameList = [self.__selDates(x) for x in dirnameList]
264 266 dirnameList = filter(lambda x:x!=None,dirnameList)
265 267
266 268 if len(dirnameList)>0:
267 269 self.status = 1
268 270 self.dirnameList = dirnameList
269 271 self.dirnameList.sort()
270 272
271 273 else:
272 274 self.status = 0
273 275 return None
274 276
275 277 def __getTimeFromData(self):
276 278 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
277 279 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
278 280 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
279 281 print '........................................'
280 282 filter_filenameList=[]
281 283 self.filenameList.sort()
282 284 for i in range(len(self.filenameList)-1):
283 285 filename=self.filenameList[i]
284 286 dir_hf_filename= filename
285 287 fp= h5py.File(dir_hf_filename,'r')
286 288 hipoc=fp['t'].value
287 289 hipoc=hipoc+self.timezone
288 290 date_time=stuffr.unix2datestr(hipoc)
289 291 fp.close()
290 292 year =int(date_time[0:4])
291 293 month=int(date_time[5:7])
292 294 dom =int(date_time[8:10])
293 295 hour =int(date_time[11:13])
294 296 min =int(date_time[14:16])
295 297 sec =int(date_time[17:19])
296 298 this_time=datetime.datetime(year,month,dom,hour,min,sec)
297 299 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
298 300 filter_filenameList.append(filename)
299 301 filter_filenameList.sort()
300 302 self.filenameList = filter_filenameList
301 303 return 1
302 304
303 305 def __getFilenameList(self):
304 306 #print "hola"
305 307 #print self.dirnameList
306 308 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
307 self.filenameList= dirList
309 self.filenameList= dirList
308 310
309 311 def __selectDataForTimes(self, online=False):
310 312
311 313 if not(self.status):
312 314 return None
315 #----------------
313 316 self.__getFilenameList()
317 #----------------
314 318 if not(online):
315 319 if not(self.all):
316 320 self.__getTimeFromData()
317 321 if len(self.filenameList)>0:
318 322 self.status=1
319 323 self.filenameList.sort()
320 324 else:
321 325 self.status=0
322 326 return None
323 327 else:
324 328 #if self.set== None:
325 329 self.filenameList=[self.filenameList[-1]]
326 330 #else:
327 331 # try:
328 332 # filename=getFileFromSet(self.path,self.ext,self.set)
329 333 # self.filenameList=self.path+"/"+filename
330 334 # except:
331 335 # self.filenameList=[self.filenameList[-1]]
332 336
333 337
334 338 def __searchFilesOffline(self,
335 339 path,
336 340 startDate,
337 341 endDate,
338 342 ext,
339 343 startTime=datetime.time(0,0,0),
340 344 endTime=datetime.time(23,59,59),
341 345 walk=True):
342 346
343 347 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
344 348
345 349 self.__checkPath()
346 350
347 351 self.__findDataForDates()
348 352 #print self.dirnameList
349 353
350 354 self.__selectDataForTimes()
351 355
352 356 for i in range(len(self.filenameList)):
353 357 print "%s"% (self.filenameList[i])
354 358
355 359 return
356 360
357 361 def __searchFilesOnline(self,
358 362 path,
359 363 expLabel= "",
360 364 ext=None,
361 365 startDate=None,
362 366 endDate=None,
363 367 walk=True,
364 368 set=None):
365 369
366 370
367
371 startDate = datetime.datetime.utcnow().date()
372 endDate = datetime.datetime.utcnow().date()
373
368 374 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
369 375
370 376 self.__checkPath()
371 377
372 fullpath=path
373
374 print "%s folder was found: " %(fullpath )
375
376 if set == None:
377 filename =getlastFileFromPath(fullpath,ext)
378 startDate= datetime.datetime.utcnow().date
379 endDate= datetime.datetime.utcnow().date()
380
381 else:
382 filename= getFileFromSet(fullpath,ext,set)
383 startDate=None
384 endDate=None
385
386 if not (filename):
387 return None,None,None,None,None
388 print "%s file was found" %(filename)
389
390 dir_hf_filename= self.path+"/"+filename
391 fp= h5py.File(dir_hf_filename,'r')
392 hipoc=fp['t'].value
393 fp.close()
394 date_time=stuffr.unix2datestr(hipoc)
395
396 year =int(date_time[0:4])
397 month=int(date_time[5:7])
398 dom =int(date_time[8:10])
399 set= int(filename[4:10])
400 self.set=set-1
378 # fullpath=path
379 #
380 # print "%s folder was found: " %(fullpath )
381 #
382 # if set == None:
383 # filename =getlastFileFromPath(fullpath,ext)
384 # startDate= datetime.datetime.utcnow().date
385 # endDate= datetime.datetime.utcnow().date()
386 #
387 # else:
388 # filename= getFileFromSet(fullpath,ext,set)
389 # startDate=None
390 # endDate=None
391 #
392 # if not (filename):
393 # return None,None,None,None,None
394 # print "%s file was found" %(filename)
395 #
396 # dir_hf_filename= self.path+"/"+filename
397 # fp= h5py.File(dir_hf_filename,'r')
398 # hipoc=fp['t'].value
399 # fp.close()
400 # date_time=stuffr.unix2datestr(hipoc)
401 #
402 # year =int(date_time[0:4])
403 # month=int(date_time[5:7])
404 # dom =int(date_time[8:10])
405 # set= int(filename[4:10])
406 # self.set=set-1
401 407 #self.dirnameList=[filename]
402 408 self.__findDataForDates(online=True)
409 self.dirnameList=[self.dirnameList[-1]]
403 410 #print self.dirnameList
404 411 self.__selectDataForTimes(online=True)
405 return fullpath,filename,year,month,dom,set
412 #return fullpath,filename,year,month,dom,set
413 return
406 414
407 415 def __setNextFile(self,online=False):
408 416 """
409 417 """
410 418 if not(online):
411 419 newFile = self.__setNextFileOffline()
412 420 else:
413 421 newFile = self.__setNextFileOnline()
414 422
415 423 if not(newFile):
416 424 return 0
417 425 return 1
418 426
419 427 def __setNextFileOffline(self):
420 428 """
421 429 """
422 430 idFile= self.fileIndex
423 431 while(True):
424 432 idFile += 1
425 433 if not (idFile < len(self.filenameList)):
426 434 self.flagNoMoreFiles = 1
427 435 print "No more Files"
428 436 return 0
429 437 filename = self.filenameList[idFile]
430 438 hfFilePointer =h5py.File(filename,'r')
431 439
432 440 epoc=hfFilePointer['t'].value
433 441 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
434 442 break
435 443
436 444 self.flagIsNewFile = 1
437 445 self.fileIndex = idFile
438 446 self.filename = filename
439 447
440 448 self.hfFilePointer = hfFilePointer
441 449 hfFilePointer.close()
442 450 self.__t0=epoc
443 451 print "Setting the file: %s"%self.filename
444 452
445 453 return 1
446 454
447 455 def __setNextFileOnline(self):
448 456 """
449 """
450
451 self.set +=1
452 if self.set>8638:
453 print "There is no file with %s "%self.set
454 return
455
456
457 """
457 458 filename = self.filenameList[0]
458 459 if self.filename_online != None:
459 460 self.__selectDataForTimes(online=True)
460 461 filename = self.filenameList[0]
461 462 while self.filename_online == filename:
462 463 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
463 464 time.sleep(self.__waitForNewFile)
464 465 self.__findDataForDates(True)
465 466 self.__selectDataForTimes(online=True)
466 467 filename = self.filenameList[0]
467 468 print filename
468 469
469 470 hfFilePointer=h5py.File(filename,'r')
470 471 self.filename_online=filename
471 472 epoc=hfFilePointer['t'].value
472 473
473 474 self.hfFilePointer=hfFilePointer
474 475 hfFilePointer.close()
475 476 self.__t0=epoc
476 477
477 478
478 479 self.flagIsNewFile = 1
479 480 self.filename = filename
480 481
481 482 print "Setting the file: %s"%self.filename
482 483 return 1
483 484
484 485 def __getExpParameters(self):
485 486 if not(self.status):
486 487 return None
487 488
488 489 def setup(self,
489 490 path = None,
490 491 startDate = None,
491 492 endDate = None,
492 493 startTime = datetime.time(0,0,0),
493 494 endTime = datetime.time(23,59,59),
494 495 set = None,
495 496 expLabel = "",
496 497 ext = None,
497 498 all=0,
498 499 timezone=0,
499 500 online = False,
500 501 delay = 60,
501 502 walk = True):
502 503 '''
503 504 In this method we should set all initial parameters.
504 505
505 506 '''
506 507 if path==None:
507 508 raise ValueError,"The path is not valid"
508 509
509 510 if ext==None:
510 511 ext = self.ext
511 512
512 513 self.timezone= timezone
513 514 self.online= online
514 515 self.all=all
515 516
516 517
517 518 if not(online):
518 519 print "Searching files in offline mode..."
519 520
520 521 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
521 522 else:
522 523 print "Searching files in online mode..."
524 self.__searchFilesOnline(path, walk)
523 525
524 for nTries in range(self.nTries):
525
526 fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
527
528 if fullpath:
529 break
530 print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
531 time.sleep(self.delay)
532 if not(fullpath):
533 print "There ins't valid files in %s" % path
534 return None
526 # for nTries in range(self.nTries):
527 #
528 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
529 #
530 # if fullpath:
531 # break
532 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
533 # time.sleep(self.delay)
534 # if not(fullpath):
535 # print "There ins't valid files in %s" % path
536 # return None
535 537
536 538
537 539 if not(self.filenameList):
538 540 print "There is no files into the folder: %s"%(path)
539 541 sys.exit(-1)
540 542
541 543 self.__getExpParameters()
542 544
543 545 self.fileIndex = -1
544 546
545 547 self.__setNextFile(online)
546 548
547 549 self.__readMetadata()
548 550
549 551 self.__setLocalVariables()
550 552
551 553 self.__setHeaderDO()
552 554 #self.profileIndex_offset= 0
553 555
554 556 #self.profileIndex = self.profileIndex_offset
555 557
556 558 self.isConfig = True
557 559
558 560 def __readMetadata(self):
559 561 self.__readHeader()
560 562
561 563
562 564 def __setLocalVariables(self):
563 565
564 566 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
565 567 #
566 568
567 569
568 570
569 571 self.profileIndex = 9999
570 572
571 573
572 574 def __setHeaderDO(self):
573 575
574 576
575 577 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
576 578
577 579 self.dataOut.systemHeaderObj = SystemHeader()
578 580
579 581 self.dataOut.type = "Voltage"
580 582
581 583 self.dataOut.data = None
582 584
583 585 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
584 586
585 587 self.dataOut.nProfiles = 1
586 588
587 589 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
588 590
589 591 self.dataOut.channelList = range(self.nChannels)
590 592
591 593 #self.dataOut.channelIndexList = None
592 594
593 595 self.dataOut.flagNoData = True
594 596
595 597 #Set to TRUE if the data is discontinuous
596 598 self.dataOut.flagDiscontinuousBlock = False
597 599
598 600 self.dataOut.utctime = None
599 601
600 602 self.dataOut.timeZone = 0
601 603
602 604 self.dataOut.dstFlag = 0
603 605
604 606 self.dataOut.errorCount = 0
605 607
606 608 self.dataOut.nCohInt = 1
607 609
608 610 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
609 611
610 612 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
611 613
612 614 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
613 615
614 616 self.dataOut.flagShiftFFT = False
615 617
616 618 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
617 619
618 620 #Time interval between profiles
619 621 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
620 622
621 623
622 624 self.dataOut.frequency = self.__frequency
623 625
624 626 self.dataOut.realtime = self.__online
625 627
626 628 def __hasNotDataInBuffer(self):
627 629
628 630 if self.profileIndex >= self.nProfiles:
629 631 return 1
630 632
631 633 return 0
632 634
633 635 def readNextBlock(self):
634 636 if not(self.__setNewBlock()):
635 637 return 0
636 638
637 639 if not(self.readBlock()):
638 640 return 0
639 641
640 642 return 1
641 643
642 644 def __setNewBlock(self):
643 645
644 646 if self.hfFilePointer==None:
645 647 return 0
646 648
647 649 if self.flagIsNewFile:
648 650 return 1
649 651
650 652 if self.profileIndex < self.nProfiles:
651 653 return 1
652 654
653 655 self.__setNextFile(self.online)
654 656
655 657 return 1
656 658
657 659
658 660
659 661 def readBlock(self):
660 662 fp=h5py.File(self.filename,'r')
661 663 #Puntero que apunta al archivo hdf5
662 664 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
663 665 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
664 666 fp.close()
665 667 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
666 668 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
667 669 self.datablock = numpy.array([ch0,ch1])
668 670 self.flagIsNewFile=0
669 671
670 672 self.profileIndex=0
671 673
672 674 return 1
673 675
674 676 def getData(self):
675 677 if self.flagNoMoreFiles:
676 678 self.dataOut.flagNoData = True
677 679 print 'Process finished'
678 680 return 0
679 681
680 682 if self.__hasNotDataInBuffer():
681 683 if not(self.readNextBlock()):
682 684 self.dataOut.flagNodata=True
683 685 return 0
684 686
685 687 ##############################
686 688 ##############################
687 689 self.dataOut.data = self.datablock[:,:,self.profileIndex]
688 690 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex+self.timezone
689 691 self.dataOut.profileIndex= self.profileIndex
690 692 self.dataOut.flagNoData=False
691 693 self.profileIndex +=1
692 694
693 695 return self.dataOut.data
694 696
695 697
696 698 def run(self, **kwargs):
697 699 '''
698 700 This method will be called many times so here you should put all your code
699 701 '''
700 702
701 703 if not self.isConfig:
702 704 self.setup(**kwargs)
703 705 self.isConfig = True
704 706 self.getData()
705 707
706 708
707 709
708 710 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now