##// END OF EJS Templates
schain
RSS
last update
Alexander Valdez -
r615:712d29968c37
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,819 +1,868
1 '''
1 '''
2 Created on Jul 3, 2014
2 Created on Jul 3, 2014
3
3
4 @author: roj-com0419
4 @author: roj-com0419
5 '''
5 '''
6
6
7 import os,sys
7 import os,sys
8 import time,datetime
8 import time,datetime
9 import h5py
9 import h5py
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import re
12 import re
13 import stuffr
13 import stuffr
14
14
15 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 from model.data.jroheaderIO import RadarControllerHeader, SystemHeader
16 from model.data.jrodata import Voltage
16 from model.data.jrodata import Voltage
17 from model.proc.jroproc_base import ProcessingUnit, Operation
17 from model.proc.jroproc_base import ProcessingUnit, Operation
18
18
19 try:
20 from gevent import sleep
21 except:
22 from time import sleep
23
19
24 def isNumber(str):
20 def isNumber(str):
25 """
21 """
26 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
22 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
27
23
28 Excepciones:
24 Excepciones:
29 Si un determinado string no puede ser convertido a numero
25 Si un determinado string no puede ser convertido a numero
30 Input:
26 Input:
31 str, string al cual se le analiza para determinar si convertible a un numero o no
27 str, string al cual se le analiza para determinar si convertible a un numero o no
32
28
33 Return:
29 Return:
34 True : si el string es uno numerico
30 True : si el string es uno numerico
35 False : no es un string numerico
31 False : no es un string numerico
36 """
32 """
37 try:
33 try:
38 float( str )
34 float( str )
39 return True
35 return True
40 except:
36 except:
41 return False
37 return False
42
38
43 def getFileFromSet(path, ext, set):
39 def getFileFromSet(path, ext, set=None):
44 validFilelist = []
40 validFilelist = []
45 fileList = os.listdir(path)
41 fileList = os.listdir(path)
46
42
43
44 if len(fileList) < 1:
45 return None
46
47 # 0 1234 567 89A BCDE
47 # 0 1234 567 89A BCDE
48 # H YYYY DDD SSS .ext
48 # H YYYY DDD SSS .ext
49
49
50 for thisFile in fileList:
50 for thisFile in fileList:
51 try:
51 try:
52 number= int(thisFile[6:16])
52 number= int(thisFile[6:16])
53
53
54 # year = int(thisFile[1:5])
54 # year = int(thisFile[1:5])
55 # doy = int(thisFile[5:8])
55 # doy = int(thisFile[5:8])
56 except:
56 except:
57 continue
57 continue
58
58
59 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
59 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
60 continue
60 continue
61
61
62 validFilelist.append(thisFile)
62 validFilelist.append(thisFile)
63 myfile = fnmatch.filter(validFilelist,'*%10.10d*'%(set))
63
64 if len(validFilelist) < 1:
65 return None
66
67 validFilelist = sorted( validFilelist, key=str.lower )
68
69 if set == None:
70 return validFilelist[-1]
71
72 print "set =" ,set
73 for thisFile in validFilelist:
74 if set <= int(thisFile[6:16]):
75 print thisFile,int(thisFile[6:16])
76 return thisFile
77
78 return validFilelist[-1]
79
80 myfile = fnmatch.filter(validFilelist,'*%10d*'%(set))
64 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
81 #myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
65
82
66 if len(myfile)!= 0:
83 if len(myfile)!= 0:
67 return myfile[0]
84 return myfile[0]
68 else:
85 else:
69 filename = '*%10.10d%s'%(set,ext.lower())
86 filename = '*%10.10d%s'%(set,ext.lower())
70 print 'the filename %s does not exist'%filename
87 print 'the filename %s does not exist'%filename
71 print '...going to the last file: '
88 print '...going to the last file: '
72
89
73 if validFilelist:
90 if validFilelist:
74 validFilelist = sorted( validFilelist, key=str.lower )
91 validFilelist = sorted( validFilelist, key=str.lower )
75 return validFilelist[-1]
92 return validFilelist[-1]
76
93
77 return None
94 return None
78
95
79 def getlastFileFromPath(path, ext):
96 def getlastFileFromPath(path, ext):
80 """
97 """
81 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
98 Depura el fileList dejando solo los que cumplan el formato de "res-xxxxxx.ext"
82 al final de la depuracion devuelve el ultimo file de la lista que quedo.
99 al final de la depuracion devuelve el ultimo file de la lista que quedo.
83
100
84 Input:
101 Input:
85 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
102 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
86 ext : extension de los files contenidos en una carpeta
103 ext : extension de los files contenidos en una carpeta
87
104
88 Return:
105 Return:
89 El ultimo file de una determinada carpeta, no se considera el path.
106 El ultimo file de una determinada carpeta, no se considera el path.
90 """
107 """
91 validFilelist = []
108 validFilelist = []
92 fileList = os.listdir(path)
109 fileList = os.listdir(path)
93
110
94 # 0 1234 567 89A BCDE
111 # 0 1234 567 89A BCDE
95 # H YYYY DDD SSS .ext
112 # H YYYY DDD SSS .ext
96
113
97 for thisFile in fileList:
114 for thisFile in fileList:
98
115
99 try:
116 try:
100 number= int(thisFile[6:16])
117 number= int(thisFile[6:16])
101 except:
118 except:
102 print "There is a file or folder with different format"
119 print "There is a file or folder with different format"
103 if not isNumber(number):
120 if not isNumber(number):
104 continue
121 continue
105
122
106 # year = thisFile[1:5]
123 # year = thisFile[1:5]
107 # if not isNumber(year):
124 # if not isNumber(year):
108 # continue
125 # continue
109
126
110 # doy = thisFile[5:8]
127 # doy = thisFile[5:8]
111 # if not isNumber(doy):
128 # if not isNumber(doy):
112 # continue
129 # continue
113
130
114 number= int(number)
131 number= int(number)
115 # year = int(year)
132 # year = int(year)
116 # doy = int(doy)
133 # doy = int(doy)
117
134
118 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
135 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
119 continue
136 continue
120
137
121
138
122 validFilelist.append(thisFile)
139 validFilelist.append(thisFile)
123
140
124
141
125 if validFilelist:
142 if validFilelist:
126 validFilelist = sorted( validFilelist, key=str.lower )
143 validFilelist = sorted( validFilelist, key=str.lower )
127 return validFilelist[-1]
144 return validFilelist[-1]
128
145
129 return None
146 return None
130
147
131
148
132
149
133 class HFReader(ProcessingUnit):
150 class HFReader(ProcessingUnit):
134 '''
151 '''
135 classdocs
152 classdocs
136 '''
153 '''
137 path = None
154 path = None
138 startDate= None
155 startDate= None
139 endDate = None
156 endDate = None
140 startTime= None
157 startTime= None
141 endTime = None
158 endTime = None
142 walk = None
159 walk = None
143 isConfig = False
160 isConfig = False
144 dataOut=None
161 dataOut=None
145 nTries = 3
162 nTries = 3
146 ext = ".hdf5"
163 ext = ".hdf5"
147
164
148 def __init__(self):
165 def __init__(self):
149 '''
166 '''
150 Constructor
167 Constructor
151 '''
168 '''
152 ProcessingUnit.__init__(self)
169 ProcessingUnit.__init__(self)
153
170
154 self.isConfig =False
171 self.isConfig =False
155
172
156 self.datablock = None
173 self.datablock = None
157
174
158 self.filename_current=None
175 self.filename_current=None
159
176
160 self.utc = 0
177 self.utc = 0
161
178
162 self.ext='.hdf5'
179 self.ext='.hdf5'
163
180
164 self.flagIsNewFile = 1
181 self.flagIsNewFile = 1
165
182
166 #-------------------------------------------------
183 #-------------------------------------------------
167 self.fileIndex=None
184 self.fileIndex=None
168
185
169 self.profileIndex_offset=None
186 self.profileIndex_offset=None
170
187
171 self.filenameList=[]
188 self.filenameList=[]
172
189
173 self.hfFilePointer= None
190 self.hfFilePointer= None
174
191
175 self.filename_online = None
192 self.filename_online = None
176
193
177 self.status=True
194 self.status=True
178
195
179 self.flagNoMoreFiles= False
196 self.flagNoMoreFiles= False
180
197
181 self.__waitForNewFile = 1
198 self.__waitForNewFile = 20
182
199
183
200
184 #--------------------------------------------------
201 #--------------------------------------------------
185
202
186 self.dataOut = self.createObjByDefault()
203 self.dataOut = self.createObjByDefault()
187
204
188
205
189 def createObjByDefault(self):
206 def createObjByDefault(self):
190
207
191 dataObj = Voltage()
208 dataObj = Voltage()
192
209
193 return dataObj
210 return dataObj
194
211
195 def setObjProperties(self):
212 def setObjProperties(self):
196
213
197 pass
214 pass
198
215
199 def getBlockDimension(self):
216 def getBlockDimension(self):
200 """
217 """
201 Obtiene la cantidad de puntos a leer por cada bloque de datos
218 Obtiene la cantidad de puntos a leer por cada bloque de datos
202
219
203 Affected:
220 Affected:
204 self.blocksize
221 self.blocksize
205
222
206 Return:
223 Return:
207 None
224 None
208 """
225 """
209 pts2read =self.nChannels*self.nHeights*self.nProfiles
226 pts2read =self.nChannels*self.nHeights*self.nProfiles
210 self.blocksize = pts2read
227 self.blocksize = pts2read
211
228
212 def __readHeader(self):
229 def __readHeader(self):
213
230
214 self.nProfiles = 100
231 self.nProfiles = 100
215 self.nHeights = 1000
232 self.nHeights = 1000
216 self.nChannels = 2
233 self.nChannels = 2
217 self.__firstHeigth=0
234 self.__firstHeigth=0
218 self.__nSamples=1000
235 self.__nSamples=1000
219 self.__deltaHeigth=1.5
236 self.__deltaHeigth=1.5
220 self.__sample_rate=1e5
237 self.__sample_rate=1e5
221 #self.__frequency=2.72e6
238 #self.__frequency=2.72e6
222 #self.__frequency=3.64e6
239 #self.__frequency=3.64e6
223 self.__frequency=None
240 self.__frequency=None
224 self.__online = False
241 self.__online = False
225 self.filename_next_set=None
242 self.filename_next_set=None
226
243
227 #print "Frequency of Operation:", self.__frequency
244 #print "Frequency of Operation:", self.__frequency
228
245
229
246
230 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
247 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
231 self.path = path
248 self.path = path
232 self.startDate = startDate
249 self.startDate = startDate
233 self.endDate = endDate
250 self.endDate = endDate
234 self.startTime = startTime
251 self.startTime = startTime
235 self.endTime = endTime
252 self.endTime = endTime
236 self.walk = walk
253 self.walk = walk
237
254
238 def __checkPath(self):
255 def __checkPath(self):
239 if os.path.exists(self.path):
256 if os.path.exists(self.path):
240 self.status=1
257 self.status=1
241 else:
258 else:
242 self.status=0
259 self.status=0
243 print 'Path %s does not exits'%self.path
260 print 'Path %s does not exits'%self.path
244 return
261 return
245 return
262 return
246
263
247 def __selDates(self, hf_dirname_format):
264 def __selDates(self, hf_dirname_format):
248 try:
265 try:
249 dir_hf_filename= self.path+"/"+hf_dirname_format
266 dir_hf_filename= self.path+"/"+hf_dirname_format
250 fp= h5py.File(dir_hf_filename,'r')
267 fp= h5py.File(dir_hf_filename,'r')
251 hipoc=fp['t'].value
268 hipoc=fp['t'].value
252 fp.close()
269 fp.close()
253 date_time=stuffr.unix2datestr(hipoc)
270 date_time=stuffr.unix2datestr(hipoc)
254 year =int(date_time[0:4])
271 year =int(date_time[0:4])
255 month=int(date_time[5:7])
272 month=int(date_time[5:7])
256 dom =int(date_time[8:10])
273 dom =int(date_time[8:10])
257 thisDate= datetime.date(year,month,dom)
274 thisDate= datetime.date(year,month,dom)
258 if (thisDate>=self.startDate and thisDate <= self.endDate):
275 if (thisDate>=self.startDate and thisDate <= self.endDate):
259 return hf_dirname_format
276 return hf_dirname_format
260 except:
277 except:
261 return None
278 return None
262
279
263 def __findDataForDates(self,online=False):
280 def __findDataForDates(self,online=False):
264 if not(self.status):
281 if not(self.status):
265 return None
282 return None
266
283
267 pat = '\d+.\d+'
284 pat = '\d+.\d+'
268 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
285 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
269 dirnameList = filter(lambda x:x!=None,dirnameList)
286 dirnameList = filter(lambda x:x!=None,dirnameList)
270 dirnameList = [x.string for x in dirnameList]
287 dirnameList = [x.string for x in dirnameList]
271 if not(online):
288 if not(online):
272
289
273 dirnameList = [self.__selDates(x) for x in dirnameList]
290 dirnameList = [self.__selDates(x) for x in dirnameList]
274 dirnameList = filter(lambda x:x!=None,dirnameList)
291 dirnameList = filter(lambda x:x!=None,dirnameList)
275
292
276 if len(dirnameList)>0:
293 if len(dirnameList)>0:
277 self.status = 1
294 self.status = 1
278 self.dirnameList = dirnameList
295 self.dirnameList = dirnameList
279 self.dirnameList.sort()
296 self.dirnameList.sort()
280
297
281 else:
298 else:
282 self.status = 0
299 self.status = 0
283 return None
300 return None
284
301
285 def __getTimeFromData(self):
302 def __getTimeFromData(self):
286 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
303 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
287 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
304 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
288 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
305 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
289 print '........................................'
306 print '........................................'
290 filter_filenameList=[]
307 filter_filenameList=[]
291 self.filenameList.sort()
308 self.filenameList.sort()
292 for i in range(len(self.filenameList)-1):
309 for i in range(len(self.filenameList)-1):
293 filename=self.filenameList[i]
310 filename=self.filenameList[i]
294 dir_hf_filename= filename
311 dir_hf_filename= filename
295 fp= h5py.File(dir_hf_filename,'r')
312 fp= h5py.File(dir_hf_filename,'r')
296 hipoc=fp['t'].value
313 hipoc=fp['t'].value
297 hipoc=hipoc+self.timezone
314 hipoc=hipoc+self.timezone
298 date_time=stuffr.unix2datestr(hipoc)
315 date_time=stuffr.unix2datestr(hipoc)
299 fp.close()
316 fp.close()
300 year =int(date_time[0:4])
317 year =int(date_time[0:4])
301 month=int(date_time[5:7])
318 month=int(date_time[5:7])
302 dom =int(date_time[8:10])
319 dom =int(date_time[8:10])
303 hour =int(date_time[11:13])
320 hour =int(date_time[11:13])
304 min =int(date_time[14:16])
321 min =int(date_time[14:16])
305 sec =int(date_time[17:19])
322 sec =int(date_time[17:19])
306 this_time=datetime.datetime(year,month,dom,hour,min,sec)
323 this_time=datetime.datetime(year,month,dom,hour,min,sec)
307 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
324 if (this_time>=startDateTime_Reader and this_time <= endDateTime_Reader):
308 filter_filenameList.append(filename)
325 filter_filenameList.append(filename)
309 filter_filenameList.sort()
326 filter_filenameList.sort()
310 self.filenameList = filter_filenameList
327 self.filenameList = filter_filenameList
311 return 1
328 return 1
312
329
313 def __getFilenameList(self):
330 def __getFilenameList(self):
314 #print "hola"
331 #print "hola"
315 #print self.dirnameList
332 #print self.dirnameList
316 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
333 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
317 self.filenameList= dirList
334 self.filenameList= dirList
318 #print self.filenameList
335 #print self.filenameList
319 #print "pase",len(self.filenameList)
336 #print "pase",len(self.filenameList)
320
337
321 def __selectDataForTimes(self, online=False):
338 def __selectDataForTimes(self, online=False):
322
339
323 if not(self.status):
340 if not(self.status):
324 return None
341 return None
325 #----------------
342 #----------------
326 self.__getFilenameList()
343 self.__getFilenameList()
327 #----------------
344 #----------------
328 if not(online):
345 if not(online):
329 if not(self.all):
346 if not(self.all):
330 self.__getTimeFromData()
347 self.__getTimeFromData()
331 if len(self.filenameList)>0:
348 if len(self.filenameList)>0:
332 self.status=1
349 self.status=1
333 self.filenameList.sort()
350 self.filenameList.sort()
334 else:
351 else:
335 self.status=0
352 self.status=0
336 return None
353 return None
337 else:
354 else:
338 if self.set== None:
355 if self.set != None:
339 filename =getlastFileFromPath(self.path,self.ext)
356
357 filename=getFileFromSet(self.path,self.ext,self.set)
340
358
341 if self.flag_nextfile==True:
359 if self.flag_nextfile==True:
342 self.dirnameList=[filename]
360 self.dirnameList=[filename]
343 fullfilename=self.path+"/"+filename
361 fullfilename=self.path+"/"+filename
344 self.filenameList=[self.filenameList[-1]]
362 self.filenameList=[fullfilename]
345 self.filename_next_set=int(filename[6:16])+10
363 self.filename_next_set=int(filename[6:16])+10
346
364
347 self.flag_nextfile=False
365 self.flag_nextfile=False
348 else:
366 else:
349 if self.filename_next_set== int(filename[6:16]):
367 print filename
368 print "PRIMERA CONDICION"
369 #if self.filename_next_set== int(filename[6:16]):
370 print "TODO BIEN"
371
372 if filename == None:
373 raise ValueError, "corregir"
374
350 self.dirnameList=[filename]
375 self.dirnameList=[filename]
351 fullfilename=self.path+"/"+filename
376 fullfilename=self.path+"/"+filename
352 self.filenameList=[self.filenameList[-1]]
377 self.filenameList=[fullfilename]
353 self.filename_next_set=int(filename[6:16])+10
378 self.filename_next_set=int(filename[6:16])+10
379 print "Setting next file",self.filename_next_set
380 self.set=int(filename[6:16])
381 if True:
382 pass
383 else:
384 print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO"
354
385
355 else:
386 else:
356 set=self.filename_next_set
357 filename=getFileFromSet(self.path,self.ext,set=set)
358 self.filename_next_set=int(filename[6:16])+10
359 if filename==None:
360 filename =getlastFileFromPath(self.path,self.ext)
387 filename =getlastFileFromPath(self.path,self.ext)
361 self.filename_next_set=int(filename[6:16])
388
389 if self.flag_nextfile==True:
362 self.dirnameList=[filename]
390 self.dirnameList=[filename]
363 fullfilename=self.path+"/"+filename
391 fullfilename=self.path+"/"+filename
364 self.filenameList=[self.filenameList[-1]]
392 self.filenameList=[self.filenameList[-1]]
393 self.filename_next_set=int(filename[6:16])+10
365
394
395 self.flag_nextfile=False
366 else:
396 else:
367 try:
368 filename=getFileFromSet(self.path,self.ext,self.set)
397 filename=getFileFromSet(self.path,self.ext,self.set)
398 print filename
399 print "PRIMERA CONDICION"
400 #if self.filename_next_set== int(filename[6:16]):
401 print "TODO BIEN"
402
403 if filename == None:
404 raise ValueError, "corregir"
405
369 self.dirnameList=[filename]
406 self.dirnameList=[filename]
370 fullfilename=self.path+"/"+filename
407 fullfilename=self.path+"/"+filename
371 self.filenameList=[fullfilename]
408 self.filenameList=[fullfilename]
372 except:
409 self.filename_next_set=int(filename[6:16])+10
373 self.filenameList=[self.filenameList[-1]]
410 print "Setting next file",self.filename_next_set
411 self.set=int(filename[6:16])
412 if True:
413 pass
414 else:
415 print "ESTOY AQUI PORQUE NO EXISTE EL SIGUIENTE ARCHIVO"
416
374
417
375
418
376 def __searchFilesOffline(self,
419 def __searchFilesOffline(self,
377 path,
420 path,
378 startDate,
421 startDate,
379 endDate,
422 endDate,
380 ext,
423 ext,
381 startTime=datetime.time(0,0,0),
424 startTime=datetime.time(0,0,0),
382 endTime=datetime.time(23,59,59),
425 endTime=datetime.time(23,59,59),
383 walk=True):
426 walk=True):
384
427
385 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
428 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
386
429
387 self.__checkPath()
430 self.__checkPath()
388
431
389 self.__findDataForDates()
432 self.__findDataForDates()
390 #print self.dirnameList
433 #print self.dirnameList
391
434
392 self.__selectDataForTimes()
435 self.__selectDataForTimes()
393
436
394 for i in range(len(self.filenameList)):
437 for i in range(len(self.filenameList)):
395 print "%s"% (self.filenameList[i])
438 print "%s"% (self.filenameList[i])
396
439
397 return
440 return
398
441
399 def __searchFilesOnline(self,
442 def __searchFilesOnline(self,
400 path,
443 path,
401 expLabel= "",
444 expLabel= "",
402 ext=None,
445 ext=None,
403 startDate=None,
446 startDate=None,
404 endDate=None,
447 endDate=None,
405 walk=True,
448 walk=True,
406 set=None):
449 set=None):
407
450
408
451
409 startDate = datetime.datetime.utcnow().date()
452 startDate = datetime.datetime.utcnow().date()
410 endDate = datetime.datetime.utcnow().date()
453 endDate = datetime.datetime.utcnow().date()
411
454
412 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
455 self.__setParameters(path=path,startDate=startDate,endDate=endDate,walk=walk)
413
456
414 self.__checkPath()
457 self.__checkPath()
415
458
416 fullpath=path
459 fullpath=path
417 print "%s folder was found: " %(fullpath )
460 print "%s folder was found: " %(fullpath )
418
461
419 if set == None:
462 if set == None:
420 self.set=None
463 self.set=None
421 filename =getlastFileFromPath(fullpath,ext)
464 filename =getlastFileFromPath(fullpath,ext)
422 startDate= datetime.datetime.utcnow().date
465 startDate= datetime.datetime.utcnow().date
423 endDate= datetime.datetime.utcnow().date()
466 endDate= datetime.datetime.utcnow().date()
424 #
467 #
425 else:
468 else:
426 filename= getFileFromSet(fullpath,ext,set)
469 filename= getFileFromSet(fullpath,ext,set)
427 startDate=None
470 startDate=None
428 endDate=None
471 endDate=None
429 #
472 #
430 if not (filename):
473 if not (filename):
431 return None,None,None,None,None
474 return None,None,None,None,None
432 #print "%s file was found" %(filename)
475 #print "%s file was found" %(filename)
433
476
434 #
477 #
435 # dir_hf_filename= self.path+"/"+filename
478 # dir_hf_filename= self.path+"/"+filename
436 # fp= h5py.File(dir_hf_filename,'r')
479 # fp= h5py.File(dir_hf_filename,'r')
437 # hipoc=fp['t'].value
480 # hipoc=fp['t'].value
438 # fp.close()
481 # fp.close()
439 # date_time=stuffr.unix2datestr(hipoc)
482 # date_time=stuffr.unix2datestr(hipoc)
440 #
483 #
441 # year =int(date_time[0:4])
484 # year =int(date_time[0:4])
442 # month=int(date_time[5:7])
485 # month=int(date_time[5:7])
443 # dom =int(date_time[8:10])
486 # dom =int(date_time[8:10])
444 # set= int(filename[4:10])
487 # set= int(filename[4:10])
445 # self.set=set-1
488 # self.set=set-1
446 #self.dirnameList=[filename]
489 #self.dirnameList=[filename]
447 filenameList= fullpath+"/"+filename
490 filenameList= fullpath+"/"+filename
448 self.dirnameList=[filename]
491 self.dirnameList=[filename]
449 self.filenameList=[filenameList]
492 self.filenameList=[filenameList]
450 self.flag_nextfile=True
493 self.flag_nextfile=True
451
494
452 #self.__findDataForDates(online=True)
495 #self.__findDataForDates(online=True)
453 #self.dirnameList=[self.dirnameList[-1]]
496 #self.dirnameList=[self.dirnameList[-1]]
454 #print self.dirnameList
497 #print self.dirnameList
455 #self.__selectDataForTimes(online=True)
498 #self.__selectDataForTimes(online=True)
456 #return fullpath,filename,year,month,dom,set
499 #return fullpath,filename,year,month,dom,set
457 return
500 return
458
501
459 def __setNextFile(self,online=False):
502 def __setNextFile(self,online=False):
460 """
503 """
461 """
504 """
462 if not(online):
505 if not(online):
463 newFile = self.__setNextFileOffline()
506 newFile = self.__setNextFileOffline()
464 else:
507 else:
465 newFile = self.__setNextFileOnline()
508 newFile = self.__setNextFileOnline()
466
509
467 if not(newFile):
510 if not(newFile):
468 return 0
511 return 0
469 return 1
512 return 1
470
513
471 def __setNextFileOffline(self):
514 def __setNextFileOffline(self):
472 """
515 """
473 """
516 """
474 idFile= self.fileIndex
517 idFile= self.fileIndex
475 while(True):
518 while(True):
476 idFile += 1
519 idFile += 1
477 if not (idFile < len(self.filenameList)):
520 if not (idFile < len(self.filenameList)):
478 self.flagNoMoreFiles = 1
521 self.flagNoMoreFiles = 1
479 print "No more Files"
522 print "No more Files"
480 return 0
523 return 0
481 filename = self.filenameList[idFile]
524 filename = self.filenameList[idFile]
482 hfFilePointer =h5py.File(filename,'r')
525 hfFilePointer =h5py.File(filename,'r')
483
526
484 epoc=hfFilePointer['t'].value
527 epoc=hfFilePointer['t'].value
485 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
528 #this_time=datetime.datetime(year,month,dom,hour,min,sec)
486 break
529 break
487
530
488 self.flagIsNewFile = 1
531 self.flagIsNewFile = 1
489 self.fileIndex = idFile
532 self.fileIndex = idFile
490 self.filename = filename
533 self.filename = filename
491
534
492 self.hfFilePointer = hfFilePointer
535 self.hfFilePointer = hfFilePointer
493 hfFilePointer.close()
536 hfFilePointer.close()
494 self.__t0=epoc
537 self.__t0=epoc
495 print "Setting the file: %s"%self.filename
538 print "Setting the file: %s"%self.filename
496
539
497 return 1
540 return 1
498
541
499 def __setNextFileOnline(self):
542 def __setNextFileOnline(self):
500 """
543 """
501 """
544 """
545 print "SOY NONE",self.set
502 if self.set==None:
546 if self.set==None:
503 pass
547 pass
504 else:
548 else:
505 self.set +=10
549 self.set +=10
506
550
507 filename = self.filenameList[0]#fullfilename
551 filename = self.filenameList[0]#fullfilename
508 if self.filename_online != None:
552 if self.filename_online != None:
509 self.__selectDataForTimes(online=True)
553 self.__selectDataForTimes(online=True)
510 filename = self.filenameList[0]
554 filename = self.filenameList[0]
511 while self.filename_online == filename:
555 while self.filename_online == filename:
512 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
556 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
513 sleep(self.__waitForNewFile)
557 time.sleep(self.__waitForNewFile)
514 #self.__findDataForDates(online=True)
558 #self.__findDataForDates(online=True)
559 self.set=self.filename_next_set
515 self.__selectDataForTimes(online=True)
560 self.__selectDataForTimes(online=True)
516 filename = self.filenameList[0]
561 filename = self.filenameList[0]
517 sizeoffile=os.path.getsize(filename)
562 sizeoffile=os.path.getsize(filename)
518
563
519 #print filename
564 #print filename
520 sizeoffile=os.path.getsize(filename)
565 sizeoffile=os.path.getsize(filename)
521 if sizeoffile<1670240:
566 if sizeoffile<1670240:
522 print "%s is not the rigth size"%filename
567 print "%s is not the rigth size"%filename
523 delay=2
568 delay=50
524 print 'waiting %d seconds for delay...'%(delay)
569 print 'waiting %d seconds for delay...'%(delay)
525 sleep(delay)
570 time.sleep(delay)
526 sizeoffile=os.path.getsize(filename)
571 sizeoffile=os.path.getsize(filename)
527 if sizeoffile<1670240:
572 if sizeoffile<1670240:
528 delay
573 delay=50
529 print 'waiting %d more seconds for delay...'%(delay)
574 print 'waiting %d more seconds for delay...'%(delay)
530 sleep(delay)
575 time.sleep(delay)
531
532
576
577 sizeoffile=os.path.getsize(filename)
578 if sizeoffile<1670240:
579 delay=50
580 print 'waiting %d more seconds for delay...'%(delay)
581 time.sleep(delay)
533
582
534 try:
583 try:
535 hfFilePointer=h5py.File(filename,'r')
584 hfFilePointer=h5py.File(filename,'r')
536
585
537 except:
586 except:
538 print "Error reading file %s"%filename
587 print "Error reading file %s"%filename
539
588
540 self.filename_online=filename
589 self.filename_online=filename
541 epoc=hfFilePointer['t'].value
590 epoc=hfFilePointer['t'].value
542
591
543 self.hfFilePointer=hfFilePointer
592 self.hfFilePointer=hfFilePointer
544 hfFilePointer.close()
593 hfFilePointer.close()
545 self.__t0=epoc
594 self.__t0=epoc
546
595
547
596
548 self.flagIsNewFile = 1
597 self.flagIsNewFile = 1
549 self.filename = filename
598 self.filename = filename
550
599
551 print "Setting the file: %s"%self.filename
600 print "Setting the file: %s"%self.filename
552 return 1
601 return 1
553
602
554 def __getExpParameters(self):
603 def __getExpParameters(self):
555 if not(self.status):
604 if not(self.status):
556 return None
605 return None
557
606
558 def setup(self,
607 def setup(self,
559 path = None,
608 path = None,
560 startDate = None,
609 startDate = None,
561 endDate = None,
610 endDate = None,
562 startTime = datetime.time(0,0,0),
611 startTime = datetime.time(0,0,0),
563 endTime = datetime.time(23,59,59),
612 endTime = datetime.time(23,59,59),
564 set = None,
613 set = None,
565 expLabel = "",
614 expLabel = "",
566 ext = None,
615 ext = None,
567 all=0,
616 all=0,
568 timezone=0,
617 timezone=0,
569 online = False,
618 online = False,
570 delay = 60,
619 delay = 60,
571 walk = True):
620 walk = True):
572 '''
621 '''
573 In this method we should set all initial parameters.
622 In this method we should set all initial parameters.
574
623
575 '''
624 '''
576 if path==None:
625 if path==None:
577 raise ValueError,"The path is not valid"
626 raise ValueError,"The path is not valid"
578
627
579 if ext==None:
628 if ext==None:
580 ext = self.ext
629 ext = self.ext
581
630
582 self.timezone= timezone
631 self.timezone= timezone
583 self.online= online
632 self.online= online
584 self.all=all
633 self.all=all
585 #if set==None:
634 #if set==None:
586
635
587 #print set
636 #print set
588 if not(online):
637 if not(online):
589 print "Searching files in offline mode..."
638 print "Searching files in offline mode..."
590
639
591 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
640 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
592 else:
641 else:
593 print "Searching files in online mode..."
642 print "Searching files in online mode..."
594 self.__searchFilesOnline(path, walk,ext,set=set)
643 self.__searchFilesOnline(path, walk,ext,set=set)
595 if set==None:
644 if set==None:
596 pass
645 pass
597 else:
646 else:
598 self.set=set-10
647 self.set=set-10
599
648
600 # for nTries in range(self.nTries):
649 # for nTries in range(self.nTries):
601 #
650 #
602 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
603 #
652 #
604 # if fullpath:
653 # if fullpath:
605 # break
654 # break
606 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
655 # print '\tWaiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
607 # time.sleep(self.delay)
656 # time.sleep(self.delay)
608 # if not(fullpath):
657 # if not(fullpath):
609 # print "There ins't valid files in %s" % path
658 # print "There ins't valid files in %s" % path
610 # return None
659 # return None
611
660
612
661
613 if not(self.filenameList):
662 if not(self.filenameList):
614 print "There is no files into the folder: %s"%(path)
663 print "There is no files into the folder: %s"%(path)
615 sys.exit(-1)
664 sys.exit(-1)
616
665
617 self.__getExpParameters()
666 self.__getExpParameters()
618
667
619
668
620 self.fileIndex = -1
669 self.fileIndex = -1
621
670
622 self.__setNextFile(online)
671 self.__setNextFile(online)
623
672
624 self.__readMetadata()
673 self.__readMetadata()
625
674
626 self.__setLocalVariables()
675 self.__setLocalVariables()
627
676
628 self.__setHeaderDO()
677 self.__setHeaderDO()
629 #self.profileIndex_offset= 0
678 #self.profileIndex_offset= 0
630
679
631 #self.profileIndex = self.profileIndex_offset
680 #self.profileIndex = self.profileIndex_offset
632
681
633 self.isConfig = True
682 self.isConfig = True
634
683
635 def __readMetadata(self):
684 def __readMetadata(self):
636 self.__readHeader()
685 self.__readHeader()
637
686
638
687
639 def __setLocalVariables(self):
688 def __setLocalVariables(self):
640
689
641 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
690 self.datablock = numpy.zeros((self.nChannels, self.nHeights,self.nProfiles), dtype = numpy.complex)
642 #
691 #
643
692
644
693
645
694
646 self.profileIndex = 9999
695 self.profileIndex = 9999
647
696
648
697
649 def __setHeaderDO(self):
698 def __setHeaderDO(self):
650
699
651
700
652 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
701 self.dataOut.radarControllerHeaderObj = RadarControllerHeader()
653
702
654 self.dataOut.systemHeaderObj = SystemHeader()
703 self.dataOut.systemHeaderObj = SystemHeader()
655
704
656
705
657 #---------------------------------------------------------
706 #---------------------------------------------------------
658 self.dataOut.systemHeaderObj.nProfiles=100
707 self.dataOut.systemHeaderObj.nProfiles=100
659 self.dataOut.systemHeaderObj.nSamples=1000
708 self.dataOut.systemHeaderObj.nSamples=1000
660
709
661
710
662 SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
711 SAMPLING_STRUCTURE=[('h0', '<f4'), ('dh', '<f4'), ('nsa', '<u4')]
663 self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
712 self.dataOut.radarControllerHeaderObj.samplingWindow=numpy.zeros((1,),SAMPLING_STRUCTURE)
664 self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
713 self.dataOut.radarControllerHeaderObj.samplingWindow['h0']=0
665 self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
714 self.dataOut.radarControllerHeaderObj.samplingWindow['dh']=1.5
666 self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
715 self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']=1000
667 self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
716 self.dataOut.radarControllerHeaderObj.nHeights=int(self.dataOut.radarControllerHeaderObj.samplingWindow['nsa'])
668 self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
717 self.dataOut.radarControllerHeaderObj.firstHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['h0']
669 self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
718 self.dataOut.radarControllerHeaderObj.deltaHeight = self.dataOut.radarControllerHeaderObj.samplingWindow['dh']
670 self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
719 self.dataOut.radarControllerHeaderObj.samplesWin = self.dataOut.radarControllerHeaderObj.samplingWindow['nsa']
671
720
672 self.dataOut.radarControllerHeaderObj.nWindows=1
721 self.dataOut.radarControllerHeaderObj.nWindows=1
673 self.dataOut.radarControllerHeaderObj.codetype=0
722 self.dataOut.radarControllerHeaderObj.codetype=0
674 self.dataOut.radarControllerHeaderObj.numTaus=0
723 self.dataOut.radarControllerHeaderObj.numTaus=0
675 #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
724 #self.dataOut.radarControllerHeaderObj.Taus = numpy.zeros((1,),'<f4')
676
725
677
726
678 #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
727 #self.dataOut.radarControllerHeaderObj.nCode=numpy.zeros((1,), '<u4')
679 #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
728 #self.dataOut.radarControllerHeaderObj.nBaud=numpy.zeros((1,), '<u4')
680 #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
729 #self.dataOut.radarControllerHeaderObj.code=numpy.zeros(0)
681
730
682 self.dataOut.radarControllerHeaderObj.code_size=0
731 self.dataOut.radarControllerHeaderObj.code_size=0
683 self.dataOut.nBaud=0
732 self.dataOut.nBaud=0
684 self.dataOut.nCode=0
733 self.dataOut.nCode=0
685 self.dataOut.nPairs=0
734 self.dataOut.nPairs=0
686
735
687
736
688 #---------------------------------------------------------
737 #---------------------------------------------------------
689
738
690 self.dataOut.type = "Voltage"
739 self.dataOut.type = "Voltage"
691
740
692 self.dataOut.data = None
741 self.dataOut.data = None
693
742
694 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
743 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
695
744
696 self.dataOut.nProfiles = 1
745 self.dataOut.nProfiles = 1
697
746
698 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
747 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
699
748
700 self.dataOut.channelList = range(self.nChannels)
749 self.dataOut.channelList = range(self.nChannels)
701
750
702 #self.dataOut.channelIndexList = None
751 #self.dataOut.channelIndexList = None
703
752
704 self.dataOut.flagNoData = True
753 self.dataOut.flagNoData = True
705
754
706 #Set to TRUE if the data is discontinuous
755 #Set to TRUE if the data is discontinuous
707 self.dataOut.flagDiscontinuousBlock = False
756 self.dataOut.flagDiscontinuousBlock = False
708
757
709 self.dataOut.utctime = None
758 self.dataOut.utctime = None
710
759
711 self.dataOut.timeZone = 0
760 self.dataOut.timeZone = self.timezone
712
761
713 self.dataOut.dstFlag = 0
762 self.dataOut.dstFlag = 0
714
763
715 self.dataOut.errorCount = 0
764 self.dataOut.errorCount = 0
716
765
717 self.dataOut.nCohInt = 1
766 self.dataOut.nCohInt = 1
718
767
719 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
768 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
720
769
721 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
770 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
722
771
723 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
772 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
724
773
725 self.dataOut.flagShiftFFT = False
774 self.dataOut.flagShiftFFT = False
726
775
727 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
776 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
728
777
729 #Time interval between profiles
778 #Time interval between profiles
730 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
779 #self.dataOut.timeInterval =self.dataOut.ippSeconds * self.dataOut.nCohInt
731
780
732
781
733 self.dataOut.frequency = self.__frequency
782 self.dataOut.frequency = self.__frequency
734
783
735 self.dataOut.realtime = self.__online
784 self.dataOut.realtime = self.__online
736
785
737 def __hasNotDataInBuffer(self):
786 def __hasNotDataInBuffer(self):
738
787
739 if self.profileIndex >= self.nProfiles:
788 if self.profileIndex >= self.nProfiles:
740 return 1
789 return 1
741
790
742 return 0
791 return 0
743
792
744 def readNextBlock(self):
793 def readNextBlock(self):
745 if not(self.__setNewBlock()):
794 if not(self.__setNewBlock()):
746 return 0
795 return 0
747
796
748 if not(self.readBlock()):
797 if not(self.readBlock()):
749 return 0
798 return 0
750
799
751 return 1
800 return 1
752
801
753 def __setNewBlock(self):
802 def __setNewBlock(self):
754
803
755 if self.hfFilePointer==None:
804 if self.hfFilePointer==None:
756 return 0
805 return 0
757
806
758 if self.flagIsNewFile:
807 if self.flagIsNewFile:
759 return 1
808 return 1
760
809
761 if self.profileIndex < self.nProfiles:
810 if self.profileIndex < self.nProfiles:
762 return 1
811 return 1
763
812
764 self.__setNextFile(self.online)
813 self.__setNextFile(self.online)
765
814
766 return 1
815 return 1
767
816
768
817
769
818
770 def readBlock(self):
819 def readBlock(self):
771 fp=h5py.File(self.filename,'r')
820 fp=h5py.File(self.filename,'r')
772 #Puntero que apunta al archivo hdf5
821 #Puntero que apunta al archivo hdf5
773 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
822 ch0=(fp['ch0']).value #Primer canal (100,1000)--(perfiles,alturas)
774 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
823 ch1=(fp['ch1']).value #Segundo canal (100,1000)--(perfiles,alturas)
775 fp.close()
824 fp.close()
776 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
825 ch0= ch0.swapaxes(0,1) #Primer canal (100,1000)--(alturas,perfiles)
777 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
826 ch1= ch1.swapaxes(0,1) #Segundo canal (100,1000)--(alturas,perfiles)
778 self.datablock = numpy.array([ch0,ch1])
827 self.datablock = numpy.array([ch0,ch1])
779 self.flagIsNewFile=0
828 self.flagIsNewFile=0
780
829
781 self.profileIndex=0
830 self.profileIndex=0
782
831
783 return 1
832 return 1
784
833
785 def getData(self):
834 def getData(self):
786 if self.flagNoMoreFiles:
835 if self.flagNoMoreFiles:
787 self.dataOut.flagNoData = True
836 self.dataOut.flagNoData = True
788 print 'Process finished'
837 print 'Process finished'
789 return 0
838 return 0
790
839
791 if self.__hasNotDataInBuffer():
840 if self.__hasNotDataInBuffer():
792 if not(self.readNextBlock()):
841 if not(self.readNextBlock()):
793 self.dataOut.flagNodata=True
842 self.dataOut.flagNodata=True
794 return 0
843 return 0
795
844
796 ##############################
845 ##############################
797 ##############################
846 ##############################
798 self.dataOut.data = self.datablock[:,:,self.profileIndex]
847 self.dataOut.data = self.datablock[:,:,self.profileIndex]
799 self.dataOut.utctime= self.__t0 + self.dataOut.ippSeconds*self.profileIndex+self.timezone
848 self.dataOut.utctime = self.__t0 + self.dataOut.ippSeconds*self.profileIndex
800 self.dataOut.profileIndex= self.profileIndex
849 self.dataOut.profileIndex= self.profileIndex
801 self.dataOut.flagNoData=False
850 self.dataOut.flagNoData=False
802 self.profileIndex +=1
851 self.profileIndex +=1
803
852
804 return self.dataOut.data
853 return self.dataOut.data
805
854
806
855
807 def run(self, **kwargs):
856 def run(self, **kwargs):
808 '''
857 '''
809 This method will be called many times so here you should put all your code
858 This method will be called many times so here you should put all your code
810 '''
859 '''
811
860
812 if not self.isConfig:
861 if not self.isConfig:
813 self.setup(**kwargs)
862 self.setup(**kwargs)
814 self.isConfig = True
863 self.isConfig = True
815 self.getData()
864 self.getData()
816
865
817
866
818
867
819
868
General Comments 0
You need to be logged in to leave comments. Login now