##// END OF EJS Templates
schain
RSS
Lectura de datos-PROYECTO HF...
Alexander Valdez -
r538:5195014fad1e
parent child
Show More
Show file before | Show file after | Hide comments
@@ -0,0 +1,500
1 '''
2 Created on Jul 3, 2014
3
4 @author: roj-com0419
5 '''
6
7 import os
8 import time,datetime
9 import os
10 import h5py
11 import re
12 import stuffr
13
14 from model.data.jrodata import *
15 from model.proc.jroproc_base import ProcessingUnit, Operation
16
17 class HFReader(ProcessingUnit):
18 '''
19 classdocs
20 '''
21 path = None
22 startDate= None
23 endDate = None
24 startTime= None
25 endTime = None
26 walk = None
27 isConfig = False
28 dataOut=None
29 ext = ".hdf5"
30
31 def __init__(self):
32 '''
33 Constructor
34 '''
35 ProcessingUnit.__init__(self)
36
37 self.isConfig =False
38
39 self.datablock = None
40
41 self.utc = 0
42
43 self.ext='.hdf5'
44
45 self.flagIsNewFile = 1
46
47 #-------------------------------------------------
48 self.fileIndex=None
49
50 self.profileIndex_offset=None
51
52 self.filenameList=[]
53
54 self.hfFilePointer= None
55
56 self.__filename_online = None
57
58 self.status=True
59
60 self.flagNoMoreFiles= False
61
62 #--------------------------------------------------
63
64 self.dataOut = self.createObjByDefault()
65
66
67 def createObjByDefault(self):
68
69 dataObj = Voltage()
70
71 return dataObj
72
73 def setObjProperties(self):
74
75 pass
76
77 def getBlockDimension(self):
78 """
79 Obtiene la cantidad de puntos a leer por cada bloque de datos
80
81 Affected:
82 self.blocksize
83
84 Return:
85 None
86 """
87 pts2read =self.nChannels*self.nHeights*self.nProfiles
88 self.blocksize = pts2read
89
90 def __readHeader(self):
91
92 self.nProfiles = 100
93 self.nHeights = 1000
94 self.nChannels = 2
95 self.__firstHeigth=0
96 self.__nSamples=1000
97 self.__deltaHeigth=1.5
98 self.__sample_rate=10e5
99 self.__frequency=2.72e6
100 self.__online = False
101
102
103 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
104 self.path = path
105 self.startDate = startDate
106 self.endDate = endDate
107 self.startTime = startTime
108 self.endTime = endTime
109 self.walk = walk
110
111 def __checkPath(self):
112 if os.path.exists(self.path):
113 self.status=1
114 else:
115 self.status=0
116 print 'Path %s does not exits'%self.path
117 return
118
119 def __selDates(self, hf_dirname_format):
120 try:
121 dir_hf_filename= self.path+"/"+hf_dirname_format
122 fp= h5py.File(dir_hf_filename,'r')
123 hipoc=fp['t'].value
124 date_time=stuffr.unix2datestr(hipoc)
125 year =int(date_time[0:4])
126 month=int(date_time[5:7])
127 dom =int(date_time[8:10])
128 thisDate= datetime.date(year,month,dom)
129 if (thisDate>=self.startDate and thisDate <= self.endDate):
130 return hf_dirname_format
131 except:
132 return None
133
134 def __findDataForDates(self,online=False):
135 if not(self.status):
136 return None
137
138 pat = '\d+.\d+'
139 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
140 dirnameList = filter(lambda x:x!=None,dirnameList)
141 dirnameList = [x.string for x in dirnameList]
142 if not(online):
143 dirnameList = [self.__selDates(x) for x in dirnameList]
144 dirnameList = filter(lambda x:x!=None,dirnameList)
145 if len(dirnameList)>0:
146 self.status = 1
147 self.dirnameList = dirnameList
148 self.dirnameList.sort()
149 else:
150 self.status = 0
151 return None
152
153 def __getTimeFromData(self):
154 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
155 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
156 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
157 print '........................................'
158 filter_filenameList=[]
159 self.filenameList.sort()
160 for i in range(len(self.filenameList)-1):
161 filename=self.filenameList[i]
162 dir_hf_filename= filename
163 fp= h5py.File(dir_hf_filename,'r')
164 hipoc=fp['t'].value
165 date_time=stuffr.unix2datestr(hipoc)
166 year =int(date_time[0:4])
167 month=int(date_time[5:7])
168 dom =int(date_time[8:10])
169 hour =int(date_time[11:13])
170 min =int(date_time[14:16])
171 sec =int(date_time[17:19])
172 this_time=datetime.datetime(year,month,dom,hour,min,sec)
173 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
174 filter_filenameList.append(filename)
175 filter_filenameList.sort()
176 self.filenameList = filter_filenameList
177 return 1
178
179 def __getFilenameList(self):
180 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
181 self.filenameList= dirList
182
183 def __selectDataForTimes(self, online=False):
184
185 if not(self.status):
186 return none
187 self.__getFilenameList()
188 if not(online):
189 if not(self.all):
190 self.__getTimeFromData()
191 if len(self.filenameList)>0:
192 self.status=1
193 self.filenameList.sort()
194 else:
195 self.status=0
196 return None
197 else:
198 self.filenameList=[self.filenameList[-2]]
199
200 def __searchFilesOffline(self,
201 path,
202 startDate,
203 endDate,
204 ext,
205 startTime=datetime.time(0,0,0),
206 endTime=datetime.time(23,59,59),
207 walk=True):
208
209 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
210
211 self.__checkPath()
212
213 self.__findDataForDates()
214
215 self.__selectDataForTimes()
216
217 for i in range(len(self.filenameList)):
218 print "%s"% (self.filenameList[i])
219
220 return
221
222 def __searchFilesOnline(self,
223 path,
224 walk=True):
225 startDate= datetime.datetime.utcnow().date
226 endDate= datetime.datetime.utcnow().date()
227 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
228 self.__checkPath()
229 self.__findDataForDates(online=True)
230 self.dirnameList= [self.dirnameList[-1]]
231 self.__selectDataForTimes(online=True)
232 return
233
234 def __setNextFile(self,online=False):
235 """
236 """
237 if not(online):
238 newFile = self.__setNextFileOffline()
239 else:
240 newFile = self.__setNextFileOnline()
241
242 if not(newFile):
243 return 0
244 return 1
245
246 def __setNextFileOffline(self):
247 """
248 """
249 idFile= self.fileIndex
250 while(True):
251 idFile += 1
252 if not (idFile < len(self.filenameList)):
253 self.flagNoMoreFiles = 1
254 print "No more Files"
255 return 0
256 filename = self.filenameList[idFile]
257 hfFilePointer =h5py.File(filename,'r')
258
259 epoc=hfFilePointer['t'].value
260 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
261 break
262
263 self.flagIsNewFile = 1
264 self.fileIndex = idFile
265 self.filename = filename
266
267 self.hfFilePointer = hfFilePointer
268 self.__t0=epoc
269
270
271 print "Setting the file: %s"%self.filename
272
273
274
275 return 1
276
277 def __setNextFileOnline(self):
278 """
279 """
280 filename = self.filenameList[0]
281 if self.__filename_online != None:
282 self.__selecDataForTimes(online=True)
283 filename = self.filenameList[0]
284 while self.__filename_online == filename:
285 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
286 time.sleep(self.__waitForNewFile)
287 self.__selectDataForTimes(online=True)
288 filename = self.filenameList[0]
289
290 self.__filename_online=filename
291 self.hfFilePointer=h5py.File(filename,'r')
292 self.flagIsNewFile = 1
293 self.filename = filename
294
295 print "Setting the file: %s"%self.filename
296 return 1
297
298 def __getExpParameters(self):
299 if not(self.status):
300 return None
301
302 fp=self.hfFilePointer
303
304 def setup(self, path = None,
305 startDate = None,
306 endDate = None,
307 startTime = datetime.time(0,0,0),
308 endTime = datetime.time(23,59,59),
309 set = None,
310 expLabel = "",
311 ext = None,
312 all=0,
313 timezone='ut',
314 online = False,
315 delay = 60,
316 walk = True):
317 '''
318 In this method we should set all initial parameters.
319
320 '''
321 self.timezone= timezone
322 self.online= online
323 self.all=all
324 if ext==None:
325 ext = self.ext
326
327 if not(online):
328 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
329 else:
330 self.__searchFilesOnline(path,walk)
331
332 if not(self.filenameList):
333 print "There is no files into the folder: %s"%(path)
334 sys.exit(-1)
335
336 self.__getExpParameters()
337
338 self.fileIndex = -1
339
340 self.__setNextFile(online)
341
342 self.__readMetadata()
343
344 self.__setLocalVariables()
345
346 self.__setHeaderDO()
347 #self.profileIndex_offset= 0
348
349 #self.profileIndex = self.profileIndex_offset
350
351 self.isConfig = True
352
353 def __readMetadata(self):
354 self.__readHeader()
355
356
357 def __setLocalVariables(self):
358
359 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
360 #
361
362
363
364 self.profileIndex = 9999
365
366
367 def __setHeaderDO(self):
368
369 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
370
371 self.dataOut.systemHeaderObj = SystemHeader()
372
373 self.dataOut.type = "Voltage"
374
375 self.dataOut.data = None
376
377 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
378
379 self.dataOut.nProfiles = 1
380
381 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
382
383 self.dataOut.channelList = numpy.arange(self.nChannels)
384
385 # self.dataOut.channelIndexList = None
386
387 self.dataOut.flagNoData = True
388
389 #Set to TRUE if the data is discontinuous
390 self.dataOut.flagDiscontinuousBlock = False
391
392 self.dataOut.utctime = None
393
394 self.dataOut.timeZone = 0
395
396 self.dataOut.dstFlag = 0
397
398 self.dataOut.errorCount = 0
399
400 self.dataOut.nCohInt = 10
401
402 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
403
404 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
405
406 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
407
408 self.dataOut.flagShiftFFT = False
409
410 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
411
412 #Time interval between profiles
413 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
414
415
416 self.dataOut.frequency = self.__frequency
417
418 self.dataOut.realtime = self.__online
419
420 def __hasNotDataInBuffer(self):
421
422 if self.profileIndex >= self.nProfiles:
423 return 1
424
425 return 0
426
427 def readNextBlock(self):
428 if not(self.__setNewBlock()):
429 return 0
430
431 if not(self.readBlock()):
432 return 0
433
434 return 1
435
436 def __setNewBlock(self):
437
438 if self.hfFilePointer==None:
439 return 0
440
441 if self.flagIsNewFile:
442 return 1
443
444 if self.profileIndex < self.nProfiles:
445 return 1
446
447 self.__setNextFile(self.online)
448
449 return 1
450
451
452
453 def readBlock(self):
454 fp=self.hfFilePointer #Puntero que apunta al archivo hdf5
455 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
456 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
457 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
458 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
459 self.datablock = numpy.array([ch0,ch1])
460 self.flagIsNewFile=0
461
462 self.profileIndex=0
463
464 return 1
465
466 def getData(self):
467 if self.flagNoMoreFiles:
468 self.dataOut.flagNoData = True
469 print 'Process finished'
470 return 0
471
472 if self.__hasNotDataInBuffer():
473 if not(self.readNextBlock()):
474 self.dataOut.flagNodata=True
475 return 0
476
477 ##############################
478 ##############################
479 self.dataOut.data = self.datablock[:,:,self.profileIndex]
480 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex
481 self.dataOut.profileIndex= self.profileIndex
482 self.dataOut.flagNoData=False
483 self.profileIndex +=1
484
485 return self.dataOut.data
486
487
488 def run(self, **kwargs):
489 '''
490 This method will be called many times so here you should put all your code
491 '''
492
493 if not self.isConfig:
494 self.setup(**kwargs)
495 self.isConfig = True
496 self.getData()
497
498
499
500 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now