##// END OF EJS Templates
schain
RSS
ONLINE & SET IS READY
Alexander Valdez -
r552:7f199d9437df
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,710 +1,781
1 '''
1 '''
2 Created on Jul 3, 2014
2 Created on Jul 3, 2014
3
3
4 @author: roj-com0419
4 @author: roj-com0419
5 '''
5 '''
6
6
7 import os,sys
7 import os,sys
8 import time,datetime
8 import time,datetime
9 import h5py
9 import h5py
10 import numpy
10 import numpy
11 import fnmatch
11 import re
12 import re
12 import stuffr
13 import stuffr
13
14
14 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 from model.data.jrodata import Voltage
16 from model.data.jrodata import Voltage
16 from model.proc.jroproc_base import ProcessingUnit, Operation
17 from model.proc.jroproc_base import ProcessingUnit, Operation
17
18
18
19
19 def isNumber(str):
20 def isNumber(str):
20 """
21 """
21 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22
23
23 Excepciones:
24 Excepciones:
24 Si un determinado string no puede ser convertido a numero
25 Si un determinado string no puede ser convertido a numero
25 Input:
26 Input:
26 str, string al cual se le analiza para determinar si convertible a un numero o no
27 str, string al cual se le analiza para determinar si convertible a un numero o no
27
28
28 Return:
29 Return:
29 True : si el string es uno numerico
30 True : si el string es uno numerico
30 False : no es un string numerico
31 False : no es un string numerico
31 """
32 """
32 try:
33 try:
33 float( str )
34 float( str )
34 return True
35 return True
35 except:
36 except:
36 return False
37 return False
37
38
38 def getFileFromSet(path, ext, set):
39 def getFileFromSet(path, ext, set):
39 validFilelist = []
40 validFilelist = []
40 fileList = os.listdir(path)
41 fileList = os.listdir(path)
41
42
42 # 0 1234 567 89A BCDE
43 # 0 1234 567 89A BCDE
43 # H YYYY DDD SSS .ext
44 # H YYYY DDD SSS .ext
44
45
45 for thisFile in fileList:
46 for thisFile in fileList:
46 try:
47 try:
47 number= int(thisFile[4:10])
48 number= int(thisFile[4:10])
48
49
49 # year = int(thisFile[1:5])
50 # year = int(thisFile[1:5])
50 # doy = int(thisFile[5:8])
51 # doy = int(thisFile[5:8])
51 except:
52 except:
52 continue
53 continue
53
54
54 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
55 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
55 continue
56 continue
56
57
57 validFilelist.append(thisFile)
58 validFilelist.append(thisFile)
58 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(number))
59 myfile = fnmatch.filter(validFilelist,'*%6.6d*'%(set))
59 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
60
61
61 if len(myfile)!= 0:
62 if len(myfile)!= 0:
62 return myfile[0]
63 return myfile[0]
63 else:
64 else:
64 filename = '*%6.6d%s'%(number,ext.lower())
65 filename = '*%6.6d%s'%(set,ext.lower())
65 print 'the filename %s does not exist'%filename
66 print 'the filename %s does not exist'%filename
66 print '...going to the last file: '
67 print '...going to the last file: '
67
68
68 if validFilelist:
69 if validFilelist:
69 validFilelist = sorted( validFilelist, key=str.lower )
70 validFilelist = sorted( validFilelist, key=str.lower )
70 return validFilelist[-1]
71 return validFilelist[-1]
71
72
72 return None
73 return None
73
74
74 def getlastFileFromPath(path, ext):
75 def getlastFileFromPath(path, ext):
75 """
76 """
76 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
77 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
77 al final de la depuracion devuelve el ultimo file de la lista que quedo.
78 al final de la depuracion devuelve el ultimo file de la lista que quedo.
78
79
79 Input:
80 Input:
80 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
81 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
81 ext : extension de los files contenidos en una carpeta
82 ext : extension de los files contenidos en una carpeta
82
83
83 Return:
84 Return:
84 El ultimo file de una determinada carpeta, no se considera el path.
85 El ultimo file de una determinada carpeta, no se considera el path.
85 """
86 """
86 validFilelist = []
87 validFilelist = []
87 fileList = os.listdir(path)
88 fileList = os.listdir(path)
88
89
89 # 0 1234 567 89A BCDE
90 # 0 1234 567 89A BCDE
90 # H YYYY DDD SSS .ext
91 # H YYYY DDD SSS .ext
91
92
92 for thisFile in fileList:
93 for thisFile in fileList:
93
94
94 try:
95 try:
95 number= int(thisFile[4:10])
96 number= int(thisFile[4:10])
96 except:
97 except:
97 print "There is a file or folder with different format"
98 print "There is a file or folder with different format"
98 if not isNumber(number):
99 if not isNumber(number):
99 continue
100 continue
100
101
101 # year = thisFile[1:5]
102 # year = thisFile[1:5]
102 # if not isNumber(year):
103 # if not isNumber(year):
103 # continue
104 # continue
104
105
105 # doy = thisFile[5:8]
106 # doy = thisFile[5:8]
106 # if not isNumber(doy):
107 # if not isNumber(doy):
107 # continue
108 # continue
108
109
109 number= int(number)
110 number= int(number)
110 # year = int(year)
111 # year = int(year)
111 # doy = int(doy)
112 # doy = int(doy)
112
113
113 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
114 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
114 continue
115 continue
115
116
116
117
117 validFilelist.append(thisFile)
118 validFilelist.append(thisFile)
118
119
119
120
120 if validFilelist:
121 if validFilelist:
121 validFilelist = sorted( validFilelist, key=str.lower )
122 validFilelist = sorted( validFilelist, key=str.lower )
122 return validFilelist[-1]
123 return validFilelist[-1]
123
124
124 return None
125 return None
125
126
126
127
127
128
128 class HFReader(ProcessingUnit):
129 class HFReader(ProcessingUnit):
129 '''
130 '''
130 classdocs
131 classdocs
131 '''
132 '''
132 path = None
133 path = None
133 startDate= None
134 startDate= None
134 endDate = None
135 endDate = None
135 startTime= None
136 startTime= None
136 endTime = None
137 endTime = None
137 walk = None
138 walk = None
138 isConfig = False
139 isConfig = False
139 dataOut=None
140 dataOut=None
140 nTries = 3
141 nTries = 3
141 ext = ".hdf5"
142 ext = ".hdf5"
142
143
143 def __init__(self):
144 def __init__(self):
144 '''
145 '''
145 Constructor
146 Constructor
146 '''
147 '''
147 ProcessingUnit.__init__(self)
148 ProcessingUnit.__init__(self)
148
149
149 self.isConfig =False
150 self.isConfig =False
150
151
151 self.datablock = None
152 self.datablock = None
152
153
154 self.filename_current=None
155
153 self.utc = 0
156 self.utc = 0
154
157
155 self.ext='.hdf5'
158 self.ext='.hdf5'
156
159
157 self.flagIsNewFile = 1
160 self.flagIsNewFile = 1
158
161
159 #-------------------------------------------------
162 #-------------------------------------------------
160 self.fileIndex=None
163 self.fileIndex=None
161
164
162 self.profileIndex_offset=None
165 self.profileIndex_offset=None
163
166
164 self.filenameList=[]
167 self.filenameList=[]
165
168
166 self.hfFilePointer= None
169 self.hfFilePointer= None
167
170
168 self.filename_online = None
171 self.filename_online = None
169
172
170 self.status=True
173 self.status=True
171
174
172 self.flagNoMoreFiles= False
175 self.flagNoMoreFiles= False
173
176
174 self.__waitForNewFile = 3
177 self.__waitForNewFile = 1
175
178
176
179
177 #--------------------------------------------------
180 #--------------------------------------------------
178
181
179 self.dataOut = self.createObjByDefault()
182 self.dataOut = self.createObjByDefault()
180
183
181
184
182 def createObjByDefault(self):
185 def createObjByDefault(self):
183
186
184 dataObj = Voltage()
187 dataObj = Voltage()
185
188
186 return dataObj
189 return dataObj
187
190
188 def setObjProperties(self):
191 def setObjProperties(self):
189
192
190 pass
193 pass
191
194
192 def getBlockDimension(self):
195 def getBlockDimension(self):
193 """
196 """
194 Obtiene la cantidad de puntos a leer por cada bloque de datos
197 Obtiene la cantidad de puntos a leer por cada bloque de datos
195
198
196 Affected:
199 Affected:
197 self.blocksize
200 self.blocksize
198
201
199 Return:
202 Return:
200 None
203 None
201 """
204 """
202 pts2read =self.nChannels*self.nHeights*self.nProfiles
205 pts2read =self.nChannels*self.nHeights*self.nProfiles
203 self.blocksize = pts2read
206 self.blocksize = pts2read
204
207
205 def __readHeader(self):
208 def __readHeader(self):
206
209
207 self.nProfiles = 100
210 self.nProfiles = 100
208 self.nHeights = 1000
211 self.nHeights = 1000
209 self.nChannels = 2
212 self.nChannels = 2
210 self.__firstHeigth=0
213 self.__firstHeigth=0
211 self.__nSamples=1000
214 self.__nSamples=1000
212 self.__deltaHeigth=1.5
215 self.__deltaHeigth=1.5
213 self.__sample_rate=1e5
216 self.__sample_rate=1e5
214 #self.__frequency=2.72e6
217 #self.__frequency=2.72e6
215 #self.__frequency=3.64e6
218 #self.__frequency=3.64e6
216 self.__frequency=None
219 self.__frequency=None
217 self.__online = False
220 self.__online = False
221 self.filename_next_set=None
218
222
219 #print "Frequency of Operation:", self.__frequency
223 #print "Frequency of Operation:", self.__frequency
220
224
221
225
222 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
226 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
223 self.path = path
227 self.path = path
224 self.startDate = startDate
228 self.startDate = startDate
225 self.endDate = endDate
229 self.endDate = endDate
226 self.startTime = startTime
230 self.startTime = startTime
227 self.endTime = endTime
231 self.endTime = endTime
228 self.walk = walk
232 self.walk = walk
229
233
230 def __checkPath(self):
234 def __checkPath(self):
231 if os.path.exists(self.path):
235 if os.path.exists(self.path):
232 self.status=1
236 self.status=1
233 else:
237 else:
234 self.status=0
238 self.status=0
235 print 'Path %s does not exits'%self.path
239 print 'Path %s does not exits'%self.path
236 return
240 return
237 return
241 return
238
242
239 def __selDates(self, hf_dirname_format):
243 def __selDates(self, hf_dirname_format):
240 try:
244 try:
241 dir_hf_filename= self.path+"/"+hf_dirname_format
245 dir_hf_filename= self.path+"/"+hf_dirname_format
242 fp= h5py.File(dir_hf_filename,'r')
246 fp= h5py.File(dir_hf_filename,'r')
243 hipoc=fp['t'].value
247 hipoc=fp['t'].value
244 fp.close()
248 fp.close()
245 date_time=stuffr.unix2datestr(hipoc)
249 date_time=stuffr.unix2datestr(hipoc)
246 year =int(date_time[0:4])
250 year =int(date_time[0:4])
247 month=int(date_time[5:7])
251 month=int(date_time[5:7])
248 dom =int(date_time[8:10])
252 dom =int(date_time[8:10])
249 thisDate= datetime.date(year,month,dom)
253 thisDate= datetime.date(year,month,dom)
250 if (thisDate>=self.startDate and thisDate <= self.endDate):
254 if (thisDate>=self.startDate and thisDate <= self.endDate):
251 return hf_dirname_format
255 return hf_dirname_format
252 except:
256 except:
253 return None
257 return None
254
258
255 def __findDataForDates(self,online=False):
259 def __findDataForDates(self,online=False):
256 if not(self.status):
260 if not(self.status):
257 return None
261 return None
258
262
259 pat = '\d+.\d+'
263 pat = '\d+.\d+'
260 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
264 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
261 dirnameList = filter(lambda x:x!=None,dirnameList)
265 dirnameList = filter(lambda x:x!=None,dirnameList)
262 dirnameList = [x.string for x in dirnameList]
266 dirnameList = [x.string for x in dirnameList]
263 if not(online):
267 if not(online):
264
268
265 dirnameList = [self.__selDates(x) for x in dirnameList]
269 dirnameList = [self.__selDates(x) for x in dirnameList]
266 dirnameList = filter(lambda x:x!=None,dirnameList)
270 dirnameList = filter(lambda x:x!=None,dirnameList)
267
271
268 if len(dirnameList)>0:
272 if len(dirnameList)>0:
269 self.status = 1
273 self.status = 1
270 self.dirnameList = dirnameList
274 self.dirnameList = dirnameList
271 self.dirnameList.sort()
275 self.dirnameList.sort()
272
276
273 else:
277 else:
274 self.status = 0
278 self.status = 0
275 return None
279 return None
276
280
277 def __getTimeFromData(self):
281 def __getTimeFromData(self):
278 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
282 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
279 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
283 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
280 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
284 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
281 print '........................................'
285 print '........................................'
282 filter_filenameList=[]
286 filter_filenameList=[]
283 self.filenameList.sort()
287 self.filenameList.sort()
284 for i in range(len(self.filenameList)-1):
288 for i in range(len(self.filenameList)-1):
285 filename=self.filenameList[i]
289 filename=self.filenameList[i]
286 dir_hf_filename= filename
290 dir_hf_filename= filename
287 fp= h5py.File(dir_hf_filename,'r')
291 fp= h5py.File(dir_hf_filename,'r')
288 hipoc=fp['t'].value
292 hipoc=fp['t'].value
289 hipoc=hipoc+self.timezone
293 hipoc=hipoc+self.timezone
290 date_time=stuffr.unix2datestr(hipoc)
294 date_time=stuffr.unix2datestr(hipoc)
291 fp.close()
295 fp.close()
292 year =int(date_time[0:4])
296 year =int(date_time[0:4])
293 month=int(date_time[5:7])
297 month=int(date_time[5:7])
294 dom =int(date_time[8:10])
298 dom =int(date_time[8:10])
295 hour =int(date_time[11:13])
299 hour =int(date_time[11:13])
296 min =int(date_time[14:16])
300 min =int(date_time[14:16])
297 sec =int(date_time[17:19])
301 sec =int(date_time[17:19])
298 this_time=datetime.datetime(year,month,dom,hour,min,sec)
302 this_time=datetime.datetime(year,month,dom,hour,min,sec)
299 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
303 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
300 filter_filenameList.append(filename)
304 filter_filenameList.append(filename)
301 filter_filenameList.sort()
305 filter_filenameList.sort()
302 self.filenameList = filter_filenameList
306 self.filenameList = filter_filenameList
303 return 1
307 return 1
304
308
305 def __getFilenameList(self):
309 def __getFilenameList(self):
306 #print "hola"
310 #print "hola"
307 #print self.dirnameList
311 #print self.dirnameList
308 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
312 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
309 self.filenameList= dirList
313 self.filenameList= dirList
314 #print self.filenameList
315 #print "pase",len(self.filenameList)
310
316
311 def __selectDataForTimes(self, online=False):
317 def __selectDataForTimes(self, online=False):
312
318
313 if not(self.status):
319 if not(self.status):
314 return None
320 return None
315 #----------------
321 #----------------
316 self.__getFilenameList()
322 self.__getFilenameList()
317 #----------------
323 #----------------
318 if not(online):
324 if not(online):
319 if not(self.all):
325 if not(self.all):
320 self.__getTimeFromData()
326 self.__getTimeFromData()
321 if len(self.filenameList)>0:
327 if len(self.filenameList)>0:
322 self.status=1
328 self.status=1
323 self.filenameList.sort()
329 self.filenameList.sort()
324 else:
330 else:
325 self.status=0
331 self.status=0
326 return None
332 return None
327 else:
333 else:
328 #if self.set== None:
334 if self.set== None:
329 self.filenameList=[self.filenameList[-1]]
335 filename =getlastFileFromPath(self.path,self.ext)
330 #else:
336
331 # try:
337 if self.flag_nextfile==True:
332 # filename=getFileFromSet(self.path,self.ext,self.set)
338 self.dirnameList=[filename]
333 # self.filenameList=self.path+"/"+filename
339 fullfilename=self.path+"/"+filename
334 # except:
340 self.filenameList=[self.filenameList[-1]]
335 # self.filenameList=[self.filenameList[-1]]
341 self.filename_next_set=int(filename[4:10])+1
342
343 self.flag_nextfile=False
344 else:
345 if self.filename_next_set== int(filename[4:10]):
346 self.dirnameList=[filename]
347 fullfilename=self.path+"/"+filename
348 self.filenameList=[self.filenameList[-1]]
349 self.filename_next_set=int(filename[4:10])+1
350
351 else:
352 set=self.filename_next_set
353 filename=getFileFromSet(self.path,self.ext,set=set)
354 self.filename_next_set=int(filename[4:10])+1
355 if filename==None:
356 filename =getlastFileFromPath(self.path,self.ext)
357 self.filename_next_set=int(filename[4:10])
358 self.dirnameList=[filename]
359 fullfilename=self.path+"/"+filename
360 self.filenameList=[self.filenameList[-1]]
361
362 else:
363 try:
364 filename=getFileFromSet(self.path,self.ext,self.set)
365 self.dirnameList=[filename]
366 fullfilename=self.path+"/"+filename
367 self.filenameList=[fullfilename]
368 except:
369 self.filenameList=[self.filenameList[-1]]
336
370
337
371
338 def __searchFilesOffline(self,
372 def __searchFilesOffline(self,
339 path,
373 path,
340 startDate,
374 startDate,
341 endDate,
375 endDate,
342 ext,
376 ext,
343 startTime=datetime.time(0,0,0),
377 startTime=datetime.time(0,0,0),
344 endTime=datetime.time(23,59,59),
378 endTime=datetime.time(23,59,59),
345 walk=True):
379 walk=True):
346
380
347 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
381 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
348
382
349 self.__checkPath()
383 self.__checkPath()
350
384
351 self.__findDataForDates()
385 self.__findDataForDates()
352 #print self.dirnameList
386 #print self.dirnameList
353
387
354 self.__selectDataForTimes()
388 self.__selectDataForTimes()
355
389
356 for i in range(len(self.filenameList)):
390 for i in range(len(self.filenameList)):
357 print "%s"% (self.filenameList[i])
391 print "%s"% (self.filenameList[i])
358
392
359 return
393 return
360
394
361 def __searchFilesOnline(self,
395 def __searchFilesOnline(self,
362 path,
396 path,
363 expLabel= "",
397 expLabel= "",
364 ext=None,
398 ext=None,
365 startDate=None,
399 startDate=None,
366 endDate=None,
400 endDate=None,
367 walk=True,
401 walk=True,
368 set=None):
402 set=None):
369
403
370
404
371 startDate = datetime.datetime.utcnow().date()
405 startDate = datetime.datetime.utcnow().date()
372 endDate = datetime.datetime.utcnow().date()
406 endDate = datetime.datetime.utcnow().date()
373
407
374 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
408 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
375
409
376 self.__checkPath()
410 self.__checkPath()
377
411
378 # fullpath=path
412 fullpath=path
379 #
413 print "%s folder was found: " %(fullpath )
380 # print "%s folder was found: " %(fullpath )
414
381 #
415 if set == None:
382 # if set == None:
416 self.set=None
383 # filename =getlastFileFromPath(fullpath,ext)
417 filename =getlastFileFromPath(fullpath,ext)
384 # startDate= datetime.datetime.utcnow().date
418 startDate= datetime.datetime.utcnow().date
385 # endDate= datetime.datetime.utcnow().date()
419 endDate= datetime.datetime.utcnow().date()
386 #
420 #
387 # else:
421 else:
388 # filename= getFileFromSet(fullpath,ext,set)
422 filename= getFileFromSet(fullpath,ext,set)
389 # startDate=None
423 startDate=None
390 # endDate=None
424 endDate=None
391 #
425 #
392 # if not (filename):
426 if not (filename):
393 # return None,None,None,None,None
427 return None,None,None,None,None
394 # print "%s file was found" %(filename)
428 #print "%s file was found" %(filename)
429
395 #
430 #
396 # dir_hf_filename= self.path+"/"+filename
431 # dir_hf_filename= self.path+"/"+filename
397 # fp= h5py.File(dir_hf_filename,'r')
432 # fp= h5py.File(dir_hf_filename,'r')
398 # hipoc=fp['t'].value
433 # hipoc=fp['t'].value
399 # fp.close()
434 # fp.close()
400 # date_time=stuffr.unix2datestr(hipoc)
435 # date_time=stuffr.unix2datestr(hipoc)
401 #
436 #
402 # year =int(date_time[0:4])
437 # year =int(date_time[0:4])
403 # month=int(date_time[5:7])
438 # month=int(date_time[5:7])
404 # dom =int(date_time[8:10])
439 # dom =int(date_time[8:10])
405 # set= int(filename[4:10])
440 # set= int(filename[4:10])
406 # self.set=set-1
441 # self.set=set-1
407 #self.dirnameList=[filename]
442 #self.dirnameList=[filename]
408 self.__findDataForDates(online=True)
443 filenameList= fullpath+"/"+filename
409 self.dirnameList=[self.dirnameList[-1]]
444 self.dirnameList=[filename]
445 self.filenameList=[filenameList]
446 self.flag_nextfile=True
447
448 #self.__findDataForDates(online=True)
449 #self.dirnameList=[self.dirnameList[-1]]
410 #print self.dirnameList
450 #print self.dirnameList
411 self.__selectDataForTimes(online=True)
451 #self.__selectDataForTimes(online=True)
412 #return fullpath,filename,year,month,dom,set
452 #return fullpath,filename,year,month,dom,set
413 return
453 return
414
454
415 def __setNextFile(self,online=False):
455 def __setNextFile(self,online=False):
416 """
456 """
417 """
457 """
418 if not(online):
458 if not(online):
419 newFile = self.__setNextFileOffline()
459 newFile = self.__setNextFileOffline()
420 else:
460 else:
421 newFile = self.__setNextFileOnline()
461 newFile = self.__setNextFileOnline()
422
462
423 if not(newFile):
463 if not(newFile):
424 return 0
464 return 0
425 return 1
465 return 1
426
466
427 def __setNextFileOffline(self):
467 def __setNextFileOffline(self):
428 """
468 """
429 """
469 """
430 idFile= self.fileIndex
470 idFile= self.fileIndex
431 while(True):
471 while(True):
432 idFile += 1
472 idFile += 1
433 if not (idFile < len(self.filenameList)):
473 if not (idFile < len(self.filenameList)):
434 self.flagNoMoreFiles = 1
474 self.flagNoMoreFiles = 1
435 print "No more Files"
475 print "No more Files"
436 return 0
476 return 0
437 filename = self.filenameList[idFile]
477 filename = self.filenameList[idFile]
438 hfFilePointer =h5py.File(filename,'r')
478 hfFilePointer =h5py.File(filename,'r')
439
479
440 epoc=hfFilePointer['t'].value
480 epoc=hfFilePointer['t'].value
441 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
481 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
442 break
482 break
443
483
444 self.flagIsNewFile = 1
484 self.flagIsNewFile = 1
445 self.fileIndex = idFile
485 self.fileIndex = idFile
446 self.filename = filename
486 self.filename = filename
447
487
448 self.hfFilePointer = hfFilePointer
488 self.hfFilePointer = hfFilePointer
449 hfFilePointer.close()
489 hfFilePointer.close()
450 self.__t0=epoc
490 self.__t0=epoc
451 print "Setting the file: %s"%self.filename
491 print "Setting the file: %s"%self.filename
452
492
453 return 1
493 return 1
454
494
455 def __setNextFileOnline(self):
495 def __setNextFileOnline(self):
456 """
496 """
457 """
497 """
458 filename = self.filenameList[0]
498 if self.set==None:
499 pass
500 else:
501 self.set +=1
502
503 filename = self.filenameList[0]#fullfilename
459 if self.filename_online != None:
504 if self.filename_online != None:
460 self.__selectDataForTimes(online=True)
505 self.__selectDataForTimes(online=True)
461 filename = self.filenameList[0]
506 filename = self.filenameList[0]
462 while self.filename_online == filename:
507 while self.filename_online == filename:
463 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
508 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
464 time.sleep(self.__waitForNewFile)
509 time.sleep(self.__waitForNewFile)
465 self.__findDataForDates(True)
510 #self.__findDataForDates(online=True)
466 self.__selectDataForTimes(online=True)
511 self.__selectDataForTimes(online=True)
467 filename = self.filenameList[0]
512 filename = self.filenameList[0]
468 print filename
513 sizeoffile=os.path.getsize(filename)
514
515 #print filename
516 sizeoffile=os.path.getsize(filename)
517 if sizeoffile<1650240:
518 print "%s is not the rigth size"%filename
519 delay=2
520 print 'waiting %d seconds for delay...'%(delay)
521 time.sleep(delay)
522 sizeoffile=os.path.getsize(filename)
523 if sizeoffile<1650240:
524 delay
525 print 'waiting %d more seconds for delay...'%(delay)
526 time.sleep(delay)
527
528
469
529
470 hfFilePointer=h5py.File(filename,'r')
530 try:
531 hfFilePointer=h5py.File(filename,'r')
532
533 except:
534 print "Error reading file %s"%filename
535
471 self.filename_online=filename
536 self.filename_online=filename
472 epoc=hfFilePointer['t'].value
537 epoc=hfFilePointer['t'].value
473
538
474 self.hfFilePointer=hfFilePointer
539 self.hfFilePointer=hfFilePointer
475 hfFilePointer.close()
540 hfFilePointer.close()
476 self.__t0=epoc
541 self.__t0=epoc
477
542
478
543
479 self.flagIsNewFile = 1
544 self.flagIsNewFile = 1
480 self.filename = filename
545 self.filename = filename
481
546
482 print "Setting the file: %s"%self.filename
547 print "Setting the file: %s"%self.filename
483 return 1
548 return 1
484
549
485 def __getExpParameters(self):
550 def __getExpParameters(self):
486 if not(self.status):
551 if not(self.status):
487 return None
552 return None
488
553
489 def setup(self,
554 def setup(self,
490 path = None,
555 path = None,
491 startDate = None,
556 startDate = None,
492 endDate = None,
557 endDate = None,
493 startTime = datetime.time(0,0,0),
558 startTime = datetime.time(0,0,0),
494 endTime = datetime.time(23,59,59),
559 endTime = datetime.time(23,59,59),
495 set = None,
560 set = None,
496 expLabel = "",
561 expLabel = "",
497 ext = None,
562 ext = None,
498 all=0,
563 all=0,
499 timezone=0,
564 timezone=0,
500 online = False,
565 online = False,
501 delay = 60,
566 delay = 60,
502 walk = True):
567 walk = True):
503 '''
568 '''
504 In this method we should set all initial parameters.
569 In this method we should set all initial parameters.
505
570
506 '''
571 '''
507 if path==None:
572 if path==None:
508 raise ValueError,"The path is not valid"
573 raise ValueError,"The path is not valid"
509
574
510 if ext==None:
575 if ext==None:
511 ext = self.ext
576 ext = self.ext
512
577
513 self.timezone= timezone
578 self.timezone= timezone
514 self.online= online
579 self.online= online
515 self.all=all
580 self.all=all
516
581 #if set==None:
517
582
583 #print set
518 if not(online):
584 if not(online):
519 print "Searching files in offline mode..."
585 print "Searching files in offline mode..."
520
586
521 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
587 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
522 else:
588 else:
523 print "Searching files in online mode..."
589 print "Searching files in online mode..."
524 self.__searchFilesOnline(path, walk)
590 self.__searchFilesOnline(path, walk,ext,set=set)
591 if set==None:
592 pass
593 else:
594 self.set=set-1
525
595
526 # for nTries in range(self.nTries):
596 # for nTries in range(self.nTries):
527 #
597 #
528 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
598 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
529 #
599 #
530 # if fullpath:
600 # if fullpath:
531 # break
601 # break
532 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
602 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
533 # time.sleep(self.delay)
603 # time.sleep(self.delay)
534 # if not(fullpath):
604 # if not(fullpath):
535 # print "There ins't valid files in %s" % path
605 # print "There ins't valid files in %s" % path
536 # return None
606 # return None
537
607
538
608
539 if not(self.filenameList):
609 if not(self.filenameList):
540 print "There is no files into the folder: %s"%(path)
610 print "There is no files into the folder: %s"%(path)
541 sys.exit(-1)
611 sys.exit(-1)
542
612
543 self.__getExpParameters()
613 self.__getExpParameters()
544
614
615
545 self.fileIndex = -1
616 self.fileIndex = -1
546
617
547 self.__setNextFile(online)
618 self.__setNextFile(online)
548
619
549 self.__readMetadata()
620 self.__readMetadata()
550
621
551 self.__setLocalVariables()
622 self.__setLocalVariables()
552
623
553 self.__setHeaderDO()
624 self.__setHeaderDO()
554 #self.profileIndex_offset= 0
625 #self.profileIndex_offset= 0
555
626
556 #self.profileIndex = self.profileIndex_offset
627 #self.profileIndex = self.profileIndex_offset
557
628
558 self.isConfig = True
629 self.isConfig = True
559
630
560 def __readMetadata(self):
631 def __readMetadata(self):
561 self.__readHeader()
632 self.__readHeader()
562
633
563
634
564 def __setLocalVariables(self):
635 def __setLocalVariables(self):
565
636
566 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
637 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
567 #
638 #
568
639
569
640
570
641
571 self.profileIndex = 9999
642 self.profileIndex = 9999
572
643
573
644
574 def __setHeaderDO(self):
645 def __setHeaderDO(self):
575
646
576
647
577 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
648 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
578
649
579 self.dataOut.systemHeaderObj = SystemHeader()
650 self.dataOut.systemHeaderObj = SystemHeader()
580
651
581 self.dataOut.type = "Voltage"
652 self.dataOut.type = "Voltage"
582
653
583 self.dataOut.data = None
654 self.dataOut.data = None
584
655
585 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
656 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
586
657
587 self.dataOut.nProfiles = 1
658 self.dataOut.nProfiles = 1
588
659
589 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
660 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
590
661
591 self.dataOut.channelList = range(self.nChannels)
662 self.dataOut.channelList = range(self.nChannels)
592
663
593 #self.dataOut.channelIndexList = None
664 #self.dataOut.channelIndexList = None
594
665
595 self.dataOut.flagNoData = True
666 self.dataOut.flagNoData = True
596
667
597 #Set to TRUE if the data is discontinuous
668 #Set to TRUE if the data is discontinuous
598 self.dataOut.flagDiscontinuousBlock = False
669 self.dataOut.flagDiscontinuousBlock = False
599
670
600 self.dataOut.utctime = None
671 self.dataOut.utctime = None
601
672
602 self.dataOut.timeZone = 0
673 self.dataOut.timeZone = 0
603
674
604 self.dataOut.dstFlag = 0
675 self.dataOut.dstFlag = 0
605
676
606 self.dataOut.errorCount = 0
677 self.dataOut.errorCount = 0
607
678
608 self.dataOut.nCohInt = 1
679 self.dataOut.nCohInt = 1
609
680
610 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
681 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
611
682
612 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
683 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
613
684
614 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
685 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
615
686
616 self.dataOut.flagShiftFFT = False
687 self.dataOut.flagShiftFFT = False
617
688
618 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
689 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
619
690
620 #Time interval between profiles
691 #Time interval between profiles
621 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
692 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
622
693
623
694
624 self.dataOut.frequency = self.__frequency
695 self.dataOut.frequency = self.__frequency
625
696
626 self.dataOut.realtime = self.__online
697 self.dataOut.realtime = self.__online
627
698
628 def __hasNotDataInBuffer(self):
699 def __hasNotDataInBuffer(self):
629
700
630 if self.profileIndex >= self.nProfiles:
701 if self.profileIndex >= self.nProfiles:
631 return 1
702 return 1
632
703
633 return 0
704 return 0
634
705
635 def readNextBlock(self):
706 def readNextBlock(self):
636 if not(self.__setNewBlock()):
707 if not(self.__setNewBlock()):
637 return 0
708 return 0
638
709
639 if not(self.readBlock()):
710 if not(self.readBlock()):
640 return 0
711 return 0
641
712
642 return 1
713 return 1
643
714
644 def __setNewBlock(self):
715 def __setNewBlock(self):
645
716
646 if self.hfFilePointer==None:
717 if self.hfFilePointer==None:
647 return 0
718 return 0
648
719
649 if self.flagIsNewFile:
720 if self.flagIsNewFile:
650 return 1
721 return 1
651
722
652 if self.profileIndex < self.nProfiles:
723 if self.profileIndex < self.nProfiles:
653 return 1
724 return 1
654
725
655 self.__setNextFile(self.online)
726 self.__setNextFile(self.online)
656
727
657 return 1
728 return 1
658
729
659
730
660
731
661 def readBlock(self):
732 def readBlock(self):
662 fp=h5py.File(self.filename,'r')
733 fp=h5py.File(self.filename,'r')
663 #Puntero que apunta al archivo hdf5
734 #Puntero que apunta al archivo hdf5
664 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
735 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
665 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
736 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
666 fp.close()
737 fp.close()
667 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
738 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
668 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
739 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
669 self.datablock = numpy.array([ch0,ch1])
740 self.datablock = numpy.array([ch0,ch1])
670 self.flagIsNewFile=0
741 self.flagIsNewFile=0
671
742
672 self.profileIndex=0
743 self.profileIndex=0
673
744
674 return 1
745 return 1
675
746
676 def getData(self):
747 def getData(self):
677 if self.flagNoMoreFiles:
748 if self.flagNoMoreFiles:
678 self.dataOut.flagNoData = True
749 self.dataOut.flagNoData = True
679 print 'Process finished'
750 print 'Process finished'
680 return 0
751 return 0
681
752
682 if self.__hasNotDataInBuffer():
753 if self.__hasNotDataInBuffer():
683 if not(self.readNextBlock()):
754 if not(self.readNextBlock()):
684 self.dataOut.flagNodata=True
755 self.dataOut.flagNodata=True
685 return 0
756 return 0
686
757
687 ##############################
758 ##############################
688 ##############################
759 ##############################
689 self.dataOut.data = self.datablock[:,:,self.profileIndex]
760 self.dataOut.data = self.datablock[:,:,self.profileIndex]
690 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex+self.timezone
761 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex+self.timezone
691 self.dataOut.profileIndex= self.profileIndex
762 self.dataOut.profileIndex= self.profileIndex
692 self.dataOut.flagNoData=False
763 self.dataOut.flagNoData=False
693 self.profileIndex +=1
764 self.profileIndex +=1
694
765
695 return self.dataOut.data
766 return self.dataOut.data
696
767
697
768
698 def run(self, **kwargs):
769 def run(self, **kwargs):
699 '''
770 '''
700 This method will be called many times so here you should put all your code
771 This method will be called many times so here you should put all your code
701 '''
772 '''
702
773
703 if not self.isConfig:
774 if not self.isConfig:
704 self.setup(**kwargs)
775 self.setup(**kwargs)
705 self.isConfig = True
776 self.isConfig = True
706 self.getData()
777 self.getData()
707
778
708
779
709
780
710 No newline at end of file
781
General Comments 0
You need to be logged in to leave comments. Login now