##// END OF EJS Templates
schain
RSS
Cerrado los archivos hdf5 y corrigiendo el range para acceder a los index de los canales
Alexander Valdez -
r541:4f4ecda68756
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,500 +1,506
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-com0419
5 5 '''
6 6
7 import os
7 import os,sys
8 8 import time,datetime
9 import os
10 9 import h5py
10 import numpy
11 11 import re
12 12 import stuffr
13 13
14 from model.data.jrodata import *
14 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 from model.data.jrodata import Voltage
15 16 from model.proc.jroproc_base import ProcessingUnit, Operation
16 17
17 18 class HFReader(ProcessingUnit):
18 19 '''
19 20 classdocs
20 21 '''
21 22 path = None
22 23 startDate= None
23 24 endDate = None
24 25 startTime= None
25 26 endTime = None
26 27 walk = None
27 28 isConfig = False
28 29 dataOut=None
29 30 ext = ".hdf5"
30 31
31 32 def __init__(self):
32 33 '''
33 34 Constructor
34 35 '''
35 36 ProcessingUnit.__init__(self)
36 37
37 38 self.isConfig =False
38 39
39 40 self.datablock = None
40 41
41 42 self.utc = 0
42 43
43 44 self.ext='.hdf5'
44 45
45 46 self.flagIsNewFile = 1
46 47
47 48 #-------------------------------------------------
48 49 self.fileIndex=None
49 50
50 51 self.profileIndex_offset=None
51 52
52 53 self.filenameList=[]
53 54
54 55 self.hfFilePointer= None
55 56
56 57 self.__filename_online = None
57 58
58 59 self.status=True
59 60
60 61 self.flagNoMoreFiles= False
61 62
62 63 #--------------------------------------------------
63 64
64 65 self.dataOut = self.createObjByDefault()
65 66
66 67
67 68 def createObjByDefault(self):
68 69
69 70 dataObj = Voltage()
70 71
71 72 return dataObj
72 73
73 74 def setObjProperties(self):
74 75
75 76 pass
76 77
77 78 def getBlockDimension(self):
78 79 """
79 80 Obtiene la cantidad de puntos a leer por cada bloque de datos
80 81
81 82 Affected:
82 83 self.blocksize
83 84
84 85 Return:
85 86 None
86 87 """
87 88 pts2read =self.nChannels*self.nHeights*self.nProfiles
88 89 self.blocksize = pts2read
89 90
90 91 def __readHeader(self):
91 92
92 93 self.nProfiles = 100
93 94 self.nHeights = 1000
94 95 self.nChannels = 2
95 96 self.__firstHeigth=0
96 97 self.__nSamples=1000
97 98 self.__deltaHeigth=1.5
98 self.__sample_rate=10e5
99 self.__frequency=2.72e6
99 self.__sample_rate=1e5
100 #self.__frequency=2.72e6
101 self.__frequency=3.64e6
100 102 self.__online = False
101 103
102 104
103 105 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
104 106 self.path = path
105 107 self.startDate = startDate
106 108 self.endDate = endDate
107 109 self.startTime = startTime
108 110 self.endTime = endTime
109 111 self.walk = walk
110 112
111 113 def __checkPath(self):
112 114 if os.path.exists(self.path):
113 115 self.status=1
114 116 else:
115 117 self.status=0
116 118 print 'Path %s does not exits'%self.path
117 119 return
118 120
119 121 def __selDates(self, hf_dirname_format):
120 122 try:
121 123 dir_hf_filename= self.path+"/"+hf_dirname_format
122 124 fp= h5py.File(dir_hf_filename,'r')
123 125 hipoc=fp['t'].value
126 fp.close()
124 127 date_time=stuffr.unix2datestr(hipoc)
125 128 year =int(date_time[0:4])
126 129 month=int(date_time[5:7])
127 130 dom =int(date_time[8:10])
128 131 thisDate= datetime.date(year,month,dom)
129 132 if (thisDate>=self.startDate and thisDate <= self.endDate):
130 133 return hf_dirname_format
131 134 except:
132 135 return None
133 136
134 137 def __findDataForDates(self,online=False):
135 138 if not(self.status):
136 139 return None
137 140
138 141 pat = '\d+.\d+'
139 142 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
140 143 dirnameList = filter(lambda x:x!=None,dirnameList)
141 144 dirnameList = [x.string for x in dirnameList]
142 145 if not(online):
143 146 dirnameList = [self.__selDates(x) for x in dirnameList]
144 147 dirnameList = filter(lambda x:x!=None,dirnameList)
145 148 if len(dirnameList)>0:
146 149 self.status = 1
147 150 self.dirnameList = dirnameList
148 151 self.dirnameList.sort()
149 152 else:
150 153 self.status = 0
151 154 return None
152 155
153 156 def __getTimeFromData(self):
154 157 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
155 158 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
156 159 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
157 160 print '........................................'
158 161 filter_filenameList=[]
159 162 self.filenameList.sort()
160 163 for i in range(len(self.filenameList)-1):
161 164 filename=self.filenameList[i]
162 165 dir_hf_filename= filename
163 166 fp= h5py.File(dir_hf_filename,'r')
164 167 hipoc=fp['t'].value
165 168 date_time=stuffr.unix2datestr(hipoc)
169 fp.close()
166 170 year =int(date_time[0:4])
167 171 month=int(date_time[5:7])
168 172 dom =int(date_time[8:10])
169 173 hour =int(date_time[11:13])
170 174 min =int(date_time[14:16])
171 175 sec =int(date_time[17:19])
172 176 this_time=datetime.datetime(year,month,dom,hour,min,sec)
173 177 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
174 178 filter_filenameList.append(filename)
175 179 filter_filenameList.sort()
176 180 self.filenameList = filter_filenameList
177 181 return 1
178 182
179 183 def __getFilenameList(self):
180 184 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
181 185 self.filenameList= dirList
182 186
183 187 def __selectDataForTimes(self, online=False):
184 188
185 189 if not(self.status):
186 return none
190 return None
187 191 self.__getFilenameList()
188 192 if not(online):
189 193 if not(self.all):
190 194 self.__getTimeFromData()
191 195 if len(self.filenameList)>0:
192 196 self.status=1
193 197 self.filenameList.sort()
194 198 else:
195 199 self.status=0
196 200 return None
197 201 else:
198 202 self.filenameList=[self.filenameList[-2]]
199 203
200 204 def __searchFilesOffline(self,
201 205 path,
202 206 startDate,
203 207 endDate,
204 208 ext,
205 209 startTime=datetime.time(0,0,0),
206 210 endTime=datetime.time(23,59,59),
207 211 walk=True):
208 212
209 213 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
210 214
211 215 self.__checkPath()
212 216
213 217 self.__findDataForDates()
214 218
215 219 self.__selectDataForTimes()
216 220
217 221 for i in range(len(self.filenameList)):
218 222 print "%s"% (self.filenameList[i])
219 223
220 224 return
221 225
222 226 def __searchFilesOnline(self,
223 227 path,
224 228 walk=True):
225 229 startDate= datetime.datetime.utcnow().date
226 230 endDate= datetime.datetime.utcnow().date()
227 231 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
228 232 self.__checkPath()
229 233 self.__findDataForDates(online=True)
230 234 self.dirnameList= [self.dirnameList[-1]]
231 235 self.__selectDataForTimes(online=True)
232 236 return
233 237
234 238 def __setNextFile(self,online=False):
235 239 """
236 240 """
237 241 if not(online):
238 242 newFile = self.__setNextFileOffline()
239 243 else:
240 244 newFile = self.__setNextFileOnline()
241 245
242 246 if not(newFile):
243 247 return 0
244 248 return 1
245 249
246 250 def __setNextFileOffline(self):
247 251 """
248 252 """
249 253 idFile= self.fileIndex
250 254 while(True):
251 255 idFile += 1
252 256 if not (idFile < len(self.filenameList)):
253 257 self.flagNoMoreFiles = 1
254 258 print "No more Files"
255 259 return 0
256 260 filename = self.filenameList[idFile]
257 261 hfFilePointer =h5py.File(filename,'r')
258 262
259 263 epoc=hfFilePointer['t'].value
260 264 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
261 265 break
262 266
263 267 self.flagIsNewFile = 1
264 268 self.fileIndex = idFile
265 269 self.filename = filename
266 270
267 271 self.hfFilePointer = hfFilePointer
272 hfFilePointer.close()
268 273 self.__t0=epoc
269 274
270 275
271 276 print "Setting the file: %s"%self.filename
272 277
273 278
274 279
275 280 return 1
276 281
277 282 def __setNextFileOnline(self):
278 283 """
279 284 """
280 285 filename = self.filenameList[0]
281 286 if self.__filename_online != None:
282 287 self.__selecDataForTimes(online=True)
283 288 filename = self.filenameList[0]
284 289 while self.__filename_online == filename:
285 290 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
286 291 time.sleep(self.__waitForNewFile)
287 292 self.__selectDataForTimes(online=True)
288 293 filename = self.filenameList[0]
289 294
290 295 self.__filename_online=filename
291 296 self.hfFilePointer=h5py.File(filename,'r')
292 297 self.flagIsNewFile = 1
293 298 self.filename = filename
294 299
295 300 print "Setting the file: %s"%self.filename
296 301 return 1
297 302
298 303 def __getExpParameters(self):
299 304 if not(self.status):
300 return None
301
302 fp=self.hfFilePointer
305 return None
303 306
304 307 def setup(self, path = None,
305 308 startDate = None,
306 309 endDate = None,
307 310 startTime = datetime.time(0,0,0),
308 311 endTime = datetime.time(23,59,59),
309 312 set = None,
310 313 expLabel = "",
311 314 ext = None,
312 315 all=0,
313 timezone='ut',
316 timezone=0,
314 317 online = False,
315 318 delay = 60,
316 319 walk = True):
317 320 '''
318 321 In this method we should set all initial parameters.
319 322
320 323 '''
321 324 self.timezone= timezone
322 325 self.online= online
323 326 self.all=all
324 327 if ext==None:
325 328 ext = self.ext
326 329
327 330 if not(online):
328 331 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
329 332 else:
330 333 self.__searchFilesOnline(path,walk)
331 334
332 335 if not(self.filenameList):
333 336 print "There is no files into the folder: %s"%(path)
334 337 sys.exit(-1)
335 338
336 339 self.__getExpParameters()
337 340
338 341 self.fileIndex = -1
339 342
340 343 self.__setNextFile(online)
341 344
342 345 self.__readMetadata()
343 346
344 347 self.__setLocalVariables()
345 348
346 349 self.__setHeaderDO()
347 350 #self.profileIndex_offset= 0
348 351
349 352 #self.profileIndex = self.profileIndex_offset
350 353
351 354 self.isConfig = True
352 355
353 356 def __readMetadata(self):
354 357 self.__readHeader()
355 358
356 359
357 360 def __setLocalVariables(self):
358 361
359 362 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
360 363 #
361 364
362 365
363 366
364 367 self.profileIndex = 9999
365 368
366 369
367 370 def __setHeaderDO(self):
368 371
372
369 373 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
370 374
371 375 self.dataOut.systemHeaderObj = SystemHeader()
372 376
373 377 self.dataOut.type = "Voltage"
374 378
375 379 self.dataOut.data = None
376 380
377 381 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
378 382
379 383 self.dataOut.nProfiles = 1
380 384
381 385 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
382 386
383 self.dataOut.channelList = numpy.arange(self.nChannels)
384
385 # self.dataOut.channelIndexList = None
387 self.dataOut.channelList = range(self.nChannels)
388
389 #self.dataOut.channelIndexList = None
386 390
387 391 self.dataOut.flagNoData = True
388 392
389 393 #Set to TRUE if the data is discontinuous
390 394 self.dataOut.flagDiscontinuousBlock = False
391 395
392 396 self.dataOut.utctime = None
393 397
394 398 self.dataOut.timeZone = 0
395 399
396 400 self.dataOut.dstFlag = 0
397 401
398 402 self.dataOut.errorCount = 0
399 403
400 self.dataOut.nCohInt = 10
404 self.dataOut.nCohInt = 1
401 405
402 406 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
403 407
404 408 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
405 409
406 410 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
407 411
408 412 self.dataOut.flagShiftFFT = False
409 413
410 414 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
411 415
412 416 #Time interval between profiles
413 417 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
414 418
415 419
416 420 self.dataOut.frequency = self.__frequency
417 421
418 422 self.dataOut.realtime = self.__online
419 423
420 424 def __hasNotDataInBuffer(self):
421 425
422 426 if self.profileIndex >= self.nProfiles:
423 427 return 1
424 428
425 429 return 0
426 430
427 431 def readNextBlock(self):
428 432 if not(self.__setNewBlock()):
429 433 return 0
430 434
431 435 if not(self.readBlock()):
432 436 return 0
433 437
434 438 return 1
435 439
436 440 def __setNewBlock(self):
437 441
438 442 if self.hfFilePointer==None:
439 443 return 0
440 444
441 445 if self.flagIsNewFile:
442 446 return 1
443 447
444 448 if self.profileIndex < self.nProfiles:
445 449 return 1
446 450
447 451 self.__setNextFile(self.online)
448 452
449 453 return 1
450 454
451 455
452 456
453 457 def readBlock(self):
454 fp=self.hfFilePointer #Puntero que apunta al archivo hdf5
458 fp=h5py.File(self.filename,'r')
459 #Puntero que apunta al archivo hdf5
455 460 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
456 461 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
462 fp.close()
457 463 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
458 464 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
459 465 self.datablock = numpy.array([ch0,ch1])
460 466 self.flagIsNewFile=0
461 467
462 468 self.profileIndex=0
463 469
464 470 return 1
465 471
466 472 def getData(self):
467 473 if self.flagNoMoreFiles:
468 474 self.dataOut.flagNoData = True
469 475 print 'Process finished'
470 476 return 0
471 477
472 478 if self.__hasNotDataInBuffer():
473 479 if not(self.readNextBlock()):
474 480 self.dataOut.flagNodata=True
475 481 return 0
476 482
477 483 ##############################
478 484 ##############################
479 485 self.dataOut.data = self.datablock[:,:,self.profileIndex]
480 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex
486 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex+self.timezone
481 487 self.dataOut.profileIndex= self.profileIndex
482 488 self.dataOut.flagNoData=False
483 489 self.profileIndex +=1
484 490
485 491 return self.dataOut.data
486 492
487 493
488 494 def run(self, **kwargs):
489 495 '''
490 496 This method will be called many times so here you should put all your code
491 497 '''
492 498
493 499 if not self.isConfig:
494 500 self.setup(**kwargs)
495 501 self.isConfig = True
496 502 self.getData()
497 503
498 504
499 505
500 506 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now