##// END OF EJS Templates
schain-jc
RSS
-Functional HDF5 file writer unit...
Julio Valdez -
r543:5c87f3286f40
parent child
Show More
Show file before | Show file after | Hide comments
@@ -111,8 +111,8 class MomentsPlot(Figure):
111 channelIndexList.append(dataOut.channelList.index(channel))
111 channelIndexList.append(dataOut.channelList.index(channel))
112
112
113 factor = dataOut.normFactor
113 factor = dataOut.normFactor
114 x = dataOut.abscissaRange
114 x = dataOut.abscissaList
115 y = dataOut.heightRange
115 y = dataOut.heightList
116
116
117 z = dataOut.data_pre[channelIndexList,:,:]/factor
117 z = dataOut.data_pre[channelIndexList,:,:]/factor
118 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
118 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
@@ -452,8 +452,8 class WindProfilerPlot(Figure):
452 tmax = None
452 tmax = None
453
453
454 x = dataOut.getTimeRange1()
454 x = dataOut.getTimeRange1()
455 # y = dataOut.heightRange
455 # y = dataOut.heightList
456 y = dataOut.heightRange
456 y = dataOut.heightList
457
457
458 z = dataOut.data_output.copy()
458 z = dataOut.data_output.copy()
459 nplots = z.shape[0] #Number of wind dimensions estimated
459 nplots = z.shape[0] #Number of wind dimensions estimated
@@ -649,8 +649,8 class ParametersPlot(Figure):
649
649
650 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
650 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
651 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
651 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
652 SNRmin = None, SNRmax = None, SNRthresh = None, paramIndex = None, onlyPositive = False,
652 parameterIndex = None, onlyPositive = False,
653 zlabel = "", parameterName = "",
653 zlabel = "", parameterName = "", parameterObject = "data_param",
654 save=False, figpath='', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
654 save=False, figpath='', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
655 server=None, folder=None, username=None, password=None,
655 server=None, folder=None, username=None, password=None,
656 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
656 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
@@ -671,27 +671,25 class ParametersPlot(Figure):
671 zmax : None
671 zmax : None
672 """
672 """
673
673
674 data_param = getattr(dataOut, parameterObject)
675
674 if channelList == None:
676 if channelList == None:
675 channelIndexList = dataOut.channelIndexList
677 channelIndexList = numpy.arange(data_param.shape[0])
676 else:
678 else:
677 channelIndexList = []
679 channelIndexList = numpy.array(channelIndexList)
678 for channel in channelList:
679 if channel not in dataOut.channelList:
680 raise ValueError, "Channel %d is not in dataOut.channelList"
681 channelIndexList.append(dataOut.channelList.index(channel))
682
680
683 if timerange != None:
681 if timerange != None:
684 self.timerange = timerange
682 self.timerange = timerange
685
683
686 #tmin = None
684 #tmin = None
687 #tmax = None
685 #tmax = None
688 if paramIndex == None:
686 if parameterIndex == None:
689 paramIndex = 1
687 parameterIndex = 1
690 x = dataOut.getTimeRange1()
688 x = dataOut.getTimeRange1()
691 y = dataOut.heightRange
689 y = dataOut.heightList
692 z = dataOut.data_param[channelIndexList,paramIndex,:].copy()
690 z = data_param[channelIndexList,parameterIndex,:].copy()
693
691
694 zRange = dataOut.abscissaRange
692 zRange = dataOut.abscissaList
695 nplots = z.shape[0] #Number of wind dimensions estimated
693 nplots = z.shape[0] #Number of wind dimensions estimated
696 # thisDatetime = dataOut.datatime
694 # thisDatetime = dataOut.datatime
697 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
695 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
@@ -719,10 +717,6 class ParametersPlot(Figure):
719 if zmin == None: zmin = numpy.nanmin(zRange)
717 if zmin == None: zmin = numpy.nanmin(zRange)
720 if zmax == None: zmax = numpy.nanmax(zRange)
718 if zmax == None: zmax = numpy.nanmax(zRange)
721
719
722 if dataOut.data_SNR != None:
723 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
724 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
725
726 self.FTP_WEI = ftp_wei
720 self.FTP_WEI = ftp_wei
727 self.EXP_CODE = exp_code
721 self.EXP_CODE = exp_code
728 self.SUB_EXP_CODE = sub_exp_code
722 self.SUB_EXP_CODE = sub_exp_code
@@ -862,7 +856,7 class SpectralFittingPlot(Figure):
862 cutHeight = dataOut.heightList[heightindex]
856 cutHeight = dataOut.heightList[heightindex]
863
857
864 factor = dataOut.normFactor
858 factor = dataOut.normFactor
865 x = dataOut.abscissaRange[:-1]
859 x = dataOut.abscissaList[:-1]
866 #y = dataOut.getHeiRange()
860 #y = dataOut.getHeiRange()
867
861
868 z = dataOut.data_pre[:,:,heightindex]/factor
862 z = dataOut.data_pre[:,:,heightindex]/factor
@@ -1037,15 +1031,6 class EWDriftsPlot(Figure):
1037 zmax : None
1031 zmax : None
1038 """
1032 """
1039
1033
1040 if channelList == None:
1041 channelIndexList = dataOut.channelIndexList
1042 else:
1043 channelIndexList = []
1044 for channel in channelList:
1045 if channel not in dataOut.channelList:
1046 raise ValueError, "Channel %d is not in dataOut.channelList"
1047 channelIndexList.append(dataOut.channelList.index(channel))
1048
1049 if timerange != None:
1034 if timerange != None:
1050 self.timerange = timerange
1035 self.timerange = timerange
1051
1036
@@ -1053,7 +1038,7 class EWDriftsPlot(Figure):
1053 tmax = None
1038 tmax = None
1054
1039
1055 x = dataOut.getTimeRange1()
1040 x = dataOut.getTimeRange1()
1056 # y = dataOut.heightRange
1041 # y = dataOut.heightList
1057 y = dataOut.heightList
1042 y = dataOut.heightList
1058
1043
1059 z = dataOut.data_output
1044 z = dataOut.data_output
@@ -1081,7 +1066,7 class EWDriftsPlot(Figure):
1081
1066
1082 showprofile = False
1067 showprofile = False
1083 # thisDatetime = dataOut.datatime
1068 # thisDatetime = dataOut.datatime
1084 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
1069 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1085 title = wintitle + " EW Drifts"
1070 title = wintitle + " EW Drifts"
1086 xlabel = ""
1071 xlabel = ""
1087 ylabel = "Height (Km)"
1072 ylabel = "Height (Km)"
Show file before | Show file after | Hide comments
@@ -17,37 +17,58 class HDF5Reader(ProcessingUnit):
17
17
18 timezone = None
18 timezone = None
19
19
20 secStart = None
21
22 secEnd = None
23
20 fileIndex = None
24 fileIndex = None
21
25
22 blockIndex = None
26 blockIndex = None
23
27
28 blocksPerFile = None
29
24 path = None
30 path = None
25
31
32 #List of Files
33
34 filenameList = None
35
36 datetimeList = None
37
26 #Hdf5 File
38 #Hdf5 File
27
39
28 fpMetadata = None
40 fpMetadata = None
29
41
42 pathMeta = None
43
30 listMetaname = None
44 listMetaname = None
31
45
32 listMetadata = None
46 listMeta = None
47
48 listDataname = None
49
50 listData = None
51
52 listShapes = None
33
53
34 fp = None
54 fp = None
35
55
36 #dataOut reconstruction
56 #dataOut reconstruction
37
57
38
39 dataOut = None
58 dataOut = None
40
59
41 nChannels = None #Dimension 0
60 nRecords = None
42
61
43 nPoints = None #Dimension 1, number of Points or Parameters
44
62
45 nSamples = None #Dimension 2, number of samples or ranges
63 def __init__(self):
64 self.dataOut = self.__createObjByDefault()
65 return
46
66
67 def __createObjByDefault(self):
47
68
48 def __init__(self):
69 dataObj = Parameters()
49
70
50 return
71 return dataObj
51
72
52 def setup(self,path=None,
73 def setup(self,path=None,
53 startDate=None,
74 startDate=None,
@@ -67,10 +88,17 class HDF5Reader(ProcessingUnit):
67 # self.online = online
88 # self.online = online
68 self.path = path
89 self.path = path
69
90
91 startDateTime = datetime.datetime.combine(startDate,startTime)
92 endDateTime = datetime.datetime.combine(endDate,endTime)
93 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
94 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
95
96 self.secStart = secStart
97 self.secEnd = secEnd
70
98
71 if not(online):
99 if not(online):
72 #Busqueda de archivos offline
100 #Busqueda de archivos offline
73 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
101 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
74 else:
102 else:
75 self.__searchFilesOnline(path, walk)
103 self.__searchFilesOnline(path, walk)
76
104
@@ -97,6 +125,8 class HDF5Reader(ProcessingUnit):
97 ext,
125 ext,
98 startTime=datetime.time(0,0,0),
126 startTime=datetime.time(0,0,0),
99 endTime=datetime.time(23,59,59),
127 endTime=datetime.time(23,59,59),
128 secStart = 0,
129 secEnd = numpy.inf,
100 walk=True):
130 walk=True):
101
131
102 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
132 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
@@ -181,7 +211,7 class HDF5Reader(ProcessingUnit):
181 for file in fileList:
211 for file in fileList:
182
212
183 filename = os.path.join(thisPath,file)
213 filename = os.path.join(thisPath,file)
184 thisDatetime = self.__isFileinThisTime(filename, startTime, endTime)
214 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
185
215
186 if not(thisDatetime):
216 if not(thisDatetime):
187 continue
217 continue
@@ -204,7 +234,7 class HDF5Reader(ProcessingUnit):
204
234
205 return pathList, filenameList
235 return pathList, filenameList
206
236
207 def __isFileinThisTime(self, filename, startTime, endTime):
237 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
208 """
238 """
209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
239 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210
240
@@ -225,7 +255,6 class HDF5Reader(ProcessingUnit):
225
255
226 """
256 """
227
257
228
229 try:
258 try:
230 fp = fp = h5py.File(filename,'r')
259 fp = fp = h5py.File(filename,'r')
231 except IOError:
260 except IOError:
@@ -233,21 +262,20 class HDF5Reader(ProcessingUnit):
233 raise IOError, "The file %s can't be opened" %(filename)
262 raise IOError, "The file %s can't be opened" %(filename)
234
263
235 grp = fp['Data']
264 grp = fp['Data']
236 time = grp['time']
265 timeAux = grp['time']
237 time0 = time[:][0]
266 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
238
267
239 fp.close()
268 fp.close()
240
269
241 thisDatetime = datetime.datetime.utcfromtimestamp(time0)
242
243 if self.timezone == 'lt':
270 if self.timezone == 'lt':
244 thisDatetime = thisDatetime - datetime.timedelta(minutes = 300)
271 time0 -= 5*3600
245
272
246 thisTime = thisDatetime.time()
273 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
247
274
248 if not ((startTime <= thisTime) and (endTime > thisTime)):
275 if not (numpy.any(boolTimer)):
249 return None
276 return None
250
277
278 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
251 return thisDatetime
279 return thisDatetime
252
280
253 def __checkPath(self):
281 def __checkPath(self):
@@ -264,7 +292,6 class HDF5Reader(ProcessingUnit):
264 idFile += 1
292 idFile += 1
265
293
266 if not(idFile < len(self.filenameList)):
294 if not(idFile < len(self.filenameList)):
267 self.flagNoMoreFiles = 1
268 print "No more Files"
295 print "No more Files"
269 return 0
296 return 0
270
297
@@ -281,12 +308,57 class HDF5Reader(ProcessingUnit):
281 print "Setting the file: %s"%self.filename
308 print "Setting the file: %s"%self.filename
282
309
283 self.__readMetadata()
310 self.__readMetadata()
284
311 self.__setBlockList()
312 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
313 self.nRecords = self.fp['Data'].attrs['nRecords']
314 self.blockIndex = 0
285 return 1
315 return 1
286
316
317 def __setBlockList(self):
318 '''
319 self.fp
320 self.startDateTime
321 self.endDateTime
322
323 self.blockList
324 self.blocksPerFile
325
326 '''
327 filePointer = self.fp
328 secStart = self.secStart
329 secEnd = self.secEnd
330
331 grp = filePointer['Data']
332 timeVector = grp['time'].value.astype(numpy.float)[0]
333
334 if self.timezone == 'lt':
335 timeVector -= 5*3600
336
337 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
338
339 self.blockList = ind
340 self.blocksPerFile = len(ind)
341
342 return
343
287 def __readMetadata(self):
344 def __readMetadata(self):
345 '''
346 self.pathMeta
347
348 self.listShapes
349 self.listMetaname
350 self.listMeta
351
352 '''
353
288 grp = self.fp['Data']
354 grp = self.fp['Data']
289 self.pathMeta = os.path.join(self.path, grp.attrs['metadata'])
355 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
356
357 if pathMeta == self.pathMeta:
358 return
359 else:
360 self.pathMeta = pathMeta
361
290 filePointer = h5py.File(self.pathMeta,'r')
362 filePointer = h5py.File(self.pathMeta,'r')
291 groupPointer = filePointer['Metadata']
363 groupPointer = filePointer['Metadata']
292
364
@@ -295,81 +367,172 class HDF5Reader(ProcessingUnit):
295 for item in groupPointer.items():
367 for item in groupPointer.items():
296 name = item[0]
368 name = item[0]
297
369
298 if name=='data shape':
370 if name=='array dimensions':
299 self.nSamples = 1
371 table = groupPointer[name][:]
300 self.nPoints = 1
372 listShapes = {}
301 self.nChannels = 1
373 for shapes in table:
374 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
302 else:
375 else:
303 data = groupPointer[name][:]
376 data = groupPointer[name].value
304 listMetaname.append(name)
377 listMetaname.append(name)
305 listMetadata.append(data)
378 listMetadata.append(data)
306
379
307 if name=='type':
380 if name=='type':
308 self.__initDataOut(name)
381 self.__initDataOut(data)
309
382
310 filePointer.close()
383 filePointer.close()
311
384
312 self.listMetadata = listMetaname
385 self.listShapes = listShapes
313 self.listMetadata = listMetadata
386 self.listMetaname = listMetaname
387 self.listMeta = listMetadata
314
388
315 return
389 return
316
390
391 def __readData(self):
392 grp = self.fp['Data']
393 listdataname = []
394 listdata = []
395
396 for item in grp.items():
397 name = item[0]
398
399 if name == 'time':
400 listdataname.append('utctime')
401 timeAux = grp[name].value.astype(numpy.float)[0]
402 listdata.append(timeAux)
403 continue
404
405 listdataname.append(name)
406 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
407 listdata.append(array)
408
409 self.listDataname = listdataname
410 self.listData = listdata
411 return
412
413 def __setDataArray(self, nRecords, dataset, shapes):
414
415 nChannels = shapes[0] #Dimension 0
416
417 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
418
419 nSamples = shapes[2] #Dimension 2, number of samples or ranges
420
421 mode = shapes[3]
422
423 # if nPoints>1:
424 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
425 # else:
426 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
427 #
428 # chn = 'channel'
429 #
430 # for i in range(nChannels):
431 #
432 # data = dataset[chn + str(i)].value
433 #
434 # if nPoints>1:
435 # data = numpy.rollaxis(data,2)
436 #
437 # arrayData[:,i,:] = data
438
439 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
440 doSqueeze = False
441 if mode == 0:
442 strds = 'channel'
443 nDatas = nChannels
444 newShapes = (nRecords,nPoints,nSamples)
445 if nPoints == 1:
446 doSqueeze = True
447 axisSqueeze = 2
448 else:
449 strds = 'param'
450 nDatas = nPoints
451 newShapes = (nRecords,nChannels,nSamples)
452 if nChannels == 1:
453 doSqueeze = True
454 axisSqueeze = 1
455
456 for i in range(nDatas):
457
458 data = dataset[strds + str(i)].value
459 data = data.reshape(newShapes)
460
461 if mode == 0:
462 arrayData[:,i,:,:] = data
463 else:
464 arrayData[:,:,i,:] = data
465
466 if doSqueeze:
467 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
468
469 return arrayData
470
317 def __initDataOut(self, type):
471 def __initDataOut(self, type):
318
472
319 if 'type'=='Parameters':
473 # if type =='Parameters':
320 self.dataOut = Parameters()
474 # self.dataOut = Parameters()
321 elif 'type'=='Spectra':
475 # elif type =='Spectra':
322 self.dataOut = Spectra()
476 # self.dataOut = Spectra()
323 elif 'type'=='Voltage':
477 # elif type =='Voltage':
324 self.dataOut = Voltage()
478 # self.dataOut = Voltage()
325 elif 'type'=='Correlation':
479 # elif type =='Correlation':
326 self.dataOut = Correlation()
480 # self.dataOut = Correlation()
327
481
328 return
482 return
329
483
330 def __setDataOut(self):
484 def __setDataOut(self):
331 listMetadata = self.listMetadata
485 listMeta = self.listMeta
332 listMetaname = self.listMetaname
486 listMetaname = self.listMetaname
333 listDataname = self.listDataname
487 listDataname = self.listDataname
334 listData = self.listData
488 listData = self.listData
335
489
336 blockIndex = self.blockIndex
490 blockIndex = self.blockIndex
491 blockList = self.blockList
337
492
338 for i in range(len(listMetadata)):
493 for i in range(len(listMeta)):
339 setattr(self.dataOut,listMetaname[i],listMetadata[i])
494 setattr(self.dataOut,listMetaname[i],listMeta[i])
340
495
341 for j in range(len(listData)):
496 for j in range(len(listData)):
342 setattr(self.dataOut,listDataname[j][blockIndex,:],listData[j][blockIndex,:])
497 if listDataname[j]=='utctime':
498 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
499 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
500 continue
343
501
344 return
502 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
503
504 return self.dataOut.data_param
345
505
346 def getData(self):
506 def getData(self):
347
507
348 if self.flagNoMoreFiles:
508 # if self.flagNoMoreFiles:
509 # self.dataOut.flagNoData = True
510 # print 'Process finished'
511 # return 0
512 #
513 if self.blockIndex==self.blocksPerFile:
514 if not( self.__setNextFileOffline() ):
349 self.dataOut.flagNoData = True
515 self.dataOut.flagNoData = True
350 print 'Process finished'
351 return 0
516 return 0
352
517
353 if self.__hasNotDataInBuffer():
518 #
354 self.__setNextFile()
519 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
355
520 # self.dataOut.flagNoData = True
356
521 # return 0
357 if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
358 self.dataOut.flagNoData = True
359 return 0
360
522
523 self.__readData()
361 self.__setDataOut()
524 self.__setDataOut()
362 self.dataOut.flagNoData = False
525 self.dataOut.flagNoData = False
363
526
364 self.blockIndex += 1
527 self.blockIndex += 1
365
528
366 return self.dataOut.data
529 return
367
530
368 def run(self, **kwargs):
531 def run(self, **kwargs):
369
532
370 if not(self.isConfig):
533 if not(self.isConfig):
371 self.setup(**kwargs)
534 self.setup(**kwargs)
372 self.setObjProperties()
535 # self.setObjProperties()
373 self.isConfig = True
536 self.isConfig = True
374
537
375 self.getData()
538 self.getData()
@@ -412,11 +575,19 class HDF5Writer(Operation):
412
575
413 metadataList = None
576 metadataList = None
414
577
415 dataDim = None
578 arrayDim = None
416
579
417 tableDim = None
580 tableDim = None
418
581
419 dtype = [('arrayName', 'S10'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i')]
582 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
583
584 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
585
586 mode = None
587
588 nDatas = None #Number of datasets to be stored per array
589
590 nDims = None #Number Dimensions in each dataset
420
591
421 def __init__(self):
592 def __init__(self):
422
593
@@ -431,23 +602,31 class HDF5Writer(Operation):
431
602
432 if kwargs.has_key('ext'):
603 if kwargs.has_key('ext'):
433 self.ext = kwargs['ext']
604 self.ext = kwargs['ext']
434 else:
435 self.blocksPerFile = 10
436
605
437 if kwargs.has_key('blocksPerFile'):
606 if kwargs.has_key('blocksPerFile'):
438 self.blocksPerFile = kwargs['blocksPerFile']
607 self.blocksPerFile = kwargs['blocksPerFile']
439 else:
608 else:
440 self.blocksPerFile = 10
609 self.blocksPerFile = 10
441
610
611 self.metadataList = kwargs['metadataList']
612
613 self.dataList = kwargs['dataList']
614
442 self.dataOut = dataOut
615 self.dataOut = dataOut
443
616
444 self.metadataList = ['type','inputUnit','abscissaRange','heightRange']
617 if kwargs.has_key('mode'):
618 mode = kwargs['mode']
619
620 if type(mode) == int:
621 mode = numpy.zeros(len(self.dataList)) + mode
622 else:
623 mode = numpy.zeros(len(self.dataList))
445
624
446 self.dataList = ['data_param', 'data_error', 'data_SNR']
625 self.mode = mode
447
626
448 self.dataDim = numpy.zeros((len(self.dataList),3))
627 arrayDim = numpy.zeros((len(self.dataList),5))
449
628
450 #Data types
629 #Table dimensions
451
630
452 dtype0 = self.dtype
631 dtype0 = self.dtype
453
632
@@ -455,18 +634,29 class HDF5Writer(Operation):
455
634
456 for i in range(len(self.dataList)):
635 for i in range(len(self.dataList)):
457
636
458 dataDim = getattr(self.dataOut, self.dataList[i]).shape
637 dataAux = getattr(self.dataOut, self.dataList[i])
459
638
460 if len(dataDim) == 3:
639 if type(dataAux)==float or type(dataAux)==int:
461 self.dataDim[i,:] = numpy.array(dataDim)
640 arrayDim[i,0] = 1
462 else:
641 else:
463 self.dataDim[i,0] = numpy.array(dataDim)[0]
642 arrayDim0 = dataAux.shape
464 self.dataDim[i,2] = numpy.array(dataDim)[1]
643 arrayDim[i,0] = len(arrayDim0)
465 self.dataDim[i,1] = 1
644 arrayDim[i,4] = mode[i]
466
645
467 table = numpy.array((self.dataList[i],) + tuple(self.dataDim[i,:]),dtype = dtype0)
646 if len(arrayDim0) == 3:
647 arrayDim[i,1:-1] = numpy.array(arrayDim0)
648 elif len(arrayDim0) == 2:
649 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
650 elif len(arrayDim0) == 1:
651 arrayDim[i,3] = arrayDim0
652 elif len(arrayDim0) == 0:
653 arrayDim[i,0] = 1
654 arrayDim[i,3] = 1
655
656 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
468 tableList.append(table)
657 tableList.append(table)
469
658
659 self.arrayDim = arrayDim
470 self.tableDim = numpy.array(tableList, dtype = dtype0)
660 self.tableDim = numpy.array(tableList, dtype = dtype0)
471 self.blockIndex = 0
661 self.blockIndex = 0
472
662
@@ -535,6 +725,7 class HDF5Writer(Operation):
535 ext = self.ext
725 ext = self.ext
536 path = self.path
726 path = self.path
537 setFile = self.setFile
727 setFile = self.setFile
728 mode = self.mode
538
729
539 if self.fp != None:
730 if self.fp != None:
540 self.fp.close()
731 self.fp.close()
@@ -576,29 +767,48 class HDF5Writer(Operation):
576 grp = fp.create_group("Data")
767 grp = fp.create_group("Data")
577 grp.attrs['metadata'] = self.metaFile
768 grp.attrs['metadata'] = self.metaFile
578
769
579 grp['blocksPerFile'] = 0
770 # grp.attrs['blocksPerFile'] = 0
580
771
581 ds = []
772 ds = []
582 data = []
773 data = []
583
774
775 nDatas = numpy.zeros(len(self.dataList))
776 nDims = self.arrayDim[:,0]
777
584 for i in range(len(self.dataList)):
778 for i in range(len(self.dataList)):
585
779
780 if nDims[i]==1:
781 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,None) , chunks = True, dtype='S20')
782 ds.append(ds0)
783 data.append([])
784
785 else:
786
787 if mode[i]==0:
788 strMode = "channel"
789 nDatas[i] = self.arrayDim[i,1]
790 else:
791 strMode = "param"
792 nDatas[i] = self.arrayDim[i,2]
793
794 if nDims[i]==2:
795 nDatas[i] = self.arrayDim[i,2]
796
586 grp0 = grp.create_group(self.dataList[i])
797 grp0 = grp.create_group(self.dataList[i])
587
798
588 for j in range(int(self.dataDim[i,0])):
799 for j in range(int(nDatas[i])):
589 tableName = "channel" + str(j)
800 tableName = strMode + str(j)
590
801
591 if not(self.dataDim[i,1] == 1):
802 if nDims[i] == 3:
592 ds0 = grp0.create_dataset(tableName, (1,1,1) , chunks = True)
803 ds0 = grp0.create_dataset(tableName, (1,1,1) , maxshape=(None,None,None), chunks=True)
593 else:
804 else:
594 ds0 = grp0.create_dataset(tableName, (1,1) , chunks = True)
805 ds0 = grp0.create_dataset(tableName, (1,1) , maxshape=(None,None), chunks=True)
595
806
596 ds.append(ds0)
807 ds.append(ds0)
597 data.append([])
808 data.append([])
598
809
599 ds0 = grp.create_dataset("time", (1,) , chunks = True)
810 self.nDatas = nDatas
600 ds.append(ds0)
811 self.nDims = nDims
601 data.append([])
602
812
603 #Saving variables
813 #Saving variables
604 print 'Writing the file: %s'%filename
814 print 'Writing the file: %s'%filename
@@ -624,31 +834,46 class HDF5Writer(Operation):
624 '''
834 '''
625 data Array configured
835 data Array configured
626
836
837
838 self.data
627 '''
839 '''
628 #Creating Arrays
840 #Creating Arrays
629 data = self.data
841 data = self.data
842 nDatas = self.nDatas
843 nDims = self.nDims
844 mode = self.mode
630 ind = 0
845 ind = 0
846
631 for i in range(len(self.dataList)):
847 for i in range(len(self.dataList)):
632 dataAux = getattr(self.dataOut,self.dataList[i])
848 dataAux = getattr(self.dataOut,self.dataList[i])
633
849
634 for j in range(int(self.dataDim[i,0])):
850 if nDims[i] == 1:
851 data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
852 if not self.firsttime:
853 data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
854 ind += 1
855
856 else:
857 for j in range(int(nDatas[i])):
858 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
635 data[ind] = dataAux[j,:]
859 data[ind] = dataAux[j,:]
860 else:
861 data[ind] = dataAux[:,j,:]
636
862
637 if not(self.dataDim[i,1] == 1):
863 if nDims[i] == 3:
638 data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
864 data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
865
639 if not self.firsttime:
866 if not self.firsttime:
640 data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
867 data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
868
641 else:
869 else:
642 data[ind] = data[ind].reshape((1,data[ind].shape[0]))
870 data[ind] = data[ind].reshape((1,data[ind].shape[0]))
871
643 if not self.firsttime:
872 if not self.firsttime:
644 data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
873 data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
645 ind += 1
874 ind += 1
646
875
647 data[ind] = numpy.array([self.dataOut.utctime])
648 if not self.firsttime:
649 self.data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
650 self.data = data
876 self.data = data
651
652 return
877 return
653
878
654 def writeBlock(self):
879 def writeBlock(self):
@@ -656,12 +881,12 class HDF5Writer(Operation):
656 Saves the block in the HDF5 file
881 Saves the block in the HDF5 file
657 '''
882 '''
658 for i in range(len(self.ds)):
883 for i in range(len(self.ds)):
659 self.ds[i].shape = self.data[i].shape
884 self.ds[i].resize(self.data[i].shape)
660 self.ds[i][:] = self.data[i]
885 self.ds[i][:] = self.data[i]
661
886
662 self.blockIndex += 1
887 self.blockIndex += 1
663
888
664 self.grp.attrs.modify('blocksPerFile', self.blockIndex)
889 self.grp.attrs.modify('nRecords', self.blockIndex)
665
890
666 self.firsttime = False
891 self.firsttime = False
667 return
892 return
Show file before | Show file after | Hide comments
@@ -22,7 +22,7 class ParametersProc(ProcessingUnit):
22 def __init__(self):
22 def __init__(self):
23 ProcessingUnit.__init__(self)
23 ProcessingUnit.__init__(self)
24
24
25 self.objectDict = {}
25 # self.objectDict = {}
26 self.buffer = None
26 self.buffer = None
27 self.firstdatatime = None
27 self.firstdatatime = None
28 self.profIndex = 0
28 self.profIndex = 0
@@ -57,7 +57,7 class ParametersProc(ProcessingUnit):
57 self.dataOut.ippSeconds = self.dataIn.ippSeconds
57 self.dataOut.ippSeconds = self.dataIn.ippSeconds
58 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
58 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
59 self.dataOut.timeInterval = self.dataIn.timeInterval
59 self.dataOut.timeInterval = self.dataIn.timeInterval
60 self.dataOut.heightRange = self.dataIn.getHeiRange()
60 self.dataOut.heightList = self.dataIn.getHeiRange()
61 self.dataOut.frequency = self.dataIn.frequency
61 self.dataOut.frequency = self.dataIn.frequency
62
62
63 def run(self, nSeconds = None, nProfiles = None):
63 def run(self, nSeconds = None, nProfiles = None):
@@ -100,7 +100,7 class ParametersProc(ProcessingUnit):
100
100
101 if self.dataIn.type == "Spectra":
101 if self.dataIn.type == "Spectra":
102 self.dataOut.data_pre = self.dataIn.data_spc.copy()
102 self.dataOut.data_pre = self.dataIn.data_spc.copy()
103 self.dataOut.abscissaRange = self.dataIn.getVelRange(1)
103 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
104 self.dataOut.noise = self.dataIn.getNoise()
104 self.dataOut.noise = self.dataIn.getNoise()
105 self.dataOut.normFactor = self.dataIn.normFactor
105 self.dataOut.normFactor = self.dataIn.normFactor
106 self.dataOut.flagNoData = False
106 self.dataOut.flagNoData = False
@@ -112,17 +112,24 class ParametersProc(ProcessingUnit):
112 indR = numpy.where(lagRRange == 0)[0][0]
112 indR = numpy.where(lagRRange == 0)[0][0]
113
113
114 self.dataOut.data_pre = self.dataIn.data_corr.copy()[:,:,indR,:]
114 self.dataOut.data_pre = self.dataIn.data_corr.copy()[:,:,indR,:]
115 self.dataOut.abscissaRange = self.dataIn.getLagTRange(1)
115 self.dataOut.abscissaList = self.dataIn.getLagTRange(1)
116 self.dataOut.noise = self.dataIn.noise
116 self.dataOut.noise = self.dataIn.noise
117 self.dataOut.normFactor = self.dataIn.normFactor
117 self.dataOut.normFactor = self.dataIn.normFactor
118 self.dataOut.data_SNR = self.dataIn.SNR
118 self.dataOut.data_SNR = self.dataIn.SNR
119 self.dataOut.groupList = self.dataIn.pairsList
119 self.dataOut.groupList = self.dataIn.pairsList
120 self.dataOut.flagNoData = False
120 self.dataOut.flagNoData = False
121
121
122 #---------------------- Correlation Data ---------------------------
123
124 if self.dataIn.type == "Parameters":
125 self.dataOut.copy(self.dataIn)
126 self.dataOut.flagNoData = False
127
128 return True
122
129
123 self.__updateObjFromInput()
130 self.__updateObjFromInput()
124 self.firstdatatime = None
131 self.firstdatatime = None
125 self.dataOut.initUtcTime = self.dataIn.ltctime
132 self.dataOut.utctimeInit = self.dataIn.utctime
126 self.dataOut.outputInterval = self.dataIn.timeInterval
133 self.dataOut.outputInterval = self.dataIn.timeInterval
127
134
128 #------------------- Get Moments ----------------------------------
135 #------------------- Get Moments ----------------------------------
@@ -133,7 +140,7 class ParametersProc(ProcessingUnit):
133 Input:
140 Input:
134 channelList : simple channel list to select e.g. [2,3,7]
141 channelList : simple channel list to select e.g. [2,3,7]
135 self.dataOut.data_pre
142 self.dataOut.data_pre
136 self.dataOut.abscissaRange
143 self.dataOut.abscissaList
137 self.dataOut.noise
144 self.dataOut.noise
138
145
139 Affected:
146 Affected:
@@ -142,7 +149,7 class ParametersProc(ProcessingUnit):
142
149
143 '''
150 '''
144 data = self.dataOut.data_pre
151 data = self.dataOut.data_pre
145 absc = self.dataOut.abscissaRange[:-1]
152 absc = self.dataOut.abscissaList[:-1]
146 noise = self.dataOut.noise
153 noise = self.dataOut.noise
147
154
148 data_param = numpy.zeros((data.shape[0], 4, data.shape[2]))
155 data_param = numpy.zeros((data.shape[0], 4, data.shape[2]))
@@ -238,7 +245,7 class ParametersProc(ProcessingUnit):
238
245
239 Input:
246 Input:
240 self.dataOut.data_pre
247 self.dataOut.data_pre
241 self.dataOut.abscissaRange
248 self.dataOut.abscissaList
242 self.dataOut.noise
249 self.dataOut.noise
243 self.dataOut.normFactor
250 self.dataOut.normFactor
244 self.dataOut.data_SNR
251 self.dataOut.data_SNR
@@ -252,7 +259,7 class ParametersProc(ProcessingUnit):
252 data = self.dataOut.data_pre
259 data = self.dataOut.data_pre
253 normFactor = self.dataOut.normFactor
260 normFactor = self.dataOut.normFactor
254 nHeights = self.dataOut.nHeights
261 nHeights = self.dataOut.nHeights
255 absc = self.dataOut.abscissaRange[:-1]
262 absc = self.dataOut.abscissaList[:-1]
256 noise = self.dataOut.noise
263 noise = self.dataOut.noise
257 SNR = self.dataOut.data_SNR
264 SNR = self.dataOut.data_SNR
258 pairsList = self.dataOut.groupList
265 pairsList = self.dataOut.groupList
@@ -1201,7 +1208,11 class ParametersProc(ProcessingUnit):
1201
1208
1202 #Initial values
1209 #Initial values
1203 data_spc = self.dataIn.data_spc[coord,:,h]
1210 data_spc = self.dataIn.data_spc[coord,:,h]
1204 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants))
1211
1212 if (h>0)and(error1[3]<5):
1213 p0 = self.dataOut.data_param[i,:,h-1]
1214 else:
1215 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
1205
1216
1206 try:
1217 try:
1207 #Least Squares
1218 #Least Squares
@@ -1570,10 +1581,10 class WindProfiler(Operation):
1570 def run(self, dataOut, technique, **kwargs):
1581 def run(self, dataOut, technique, **kwargs):
1571
1582
1572 param = dataOut.data_param
1583 param = dataOut.data_param
1573 if dataOut.abscissaRange != None:
1584 if dataOut.abscissaList != None:
1574 absc = dataOut.abscissaRange[:-1]
1585 absc = dataOut.abscissaList[:-1]
1575 noise = dataOut.noise
1586 noise = dataOut.noise
1576 heightRange = dataOut.getHeiRange()
1587 heightList = dataOut.getHeiRange()
1577 SNR = dataOut.data_SNR
1588 SNR = dataOut.data_SNR
1578
1589
1579 if technique == 'DBS':
1590 if technique == 'DBS':
@@ -1602,7 +1613,9 class WindProfiler(Operation):
1602 theta_y = theta_y[arrayChannel]
1613 theta_y = theta_y[arrayChannel]
1603
1614
1604 velRadial0 = param[:,1,:] #Radial velocity
1615 velRadial0 = param[:,1,:] #Radial velocity
1605 dataOut.data_output, dataOut.heightRange, dataOut.data_SNR = self.techniqueDBS(velRadial0, theta_x, theta_y, azimuth, correctFactor, horizontalOnly, heightRange, SNR) #DBS Function
1616 dataOut.data_output, dataOut.heightList, dataOut.data_SNR = self.techniqueDBS(velRadial0, theta_x, theta_y, azimuth, correctFactor, horizontalOnly, heightList, SNR) #DBS Function
1617 dataOut.utctimeInit = dataOut.utctime
1618 dataOut.outputInterval = dataOut.timeInterval
1606
1619
1607 elif technique == 'SA':
1620 elif technique == 'SA':
1608
1621
@@ -1627,7 +1640,7 class WindProfiler(Operation):
1627 nChannels = dataOut.nChannels
1640 nChannels = dataOut.nChannels
1628
1641
1629 dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
1642 dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
1630 dataOut.initUtcTime = dataOut.ltctime
1643 dataOut.utctimeInit = dataOut.utctime
1631 dataOut.outputInterval = dataOut.timeInterval
1644 dataOut.outputInterval = dataOut.timeInterval
1632
1645
1633 elif technique == 'Meteors':
1646 elif technique == 'Meteors':
@@ -1656,7 +1669,9 class WindProfiler(Operation):
1656 if self.__isConfig == False:
1669 if self.__isConfig == False:
1657 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
1670 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
1658 #Get Initial LTC time
1671 #Get Initial LTC time
1659 self.__initime = (dataOut.datatime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1672 self.__initime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
1673 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1674
1660 self.__isConfig = True
1675 self.__isConfig = True
1661
1676
1662 if self.__buffer == None:
1677 if self.__buffer == None:
@@ -1666,13 +1681,14 class WindProfiler(Operation):
1666 else:
1681 else:
1667 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1682 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1668
1683
1669 self.__checkTime(dataOut.ltctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1684 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1670
1685
1671 if self.__dataReady:
1686 if self.__dataReady:
1672 dataOut.initUtcTime = self.__initime
1687 dataOut.utctimeInit = self.__initime
1673 self.__initime = self.__initime + dataOut.outputInterval #to erase time offset
1688
1689 self.__initime += dataOut.outputInterval #to erase time offset
1674
1690
1675 dataOut.data_output, dataOut.heightRange = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
1691 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
1676 dataOut.flagNoData = False
1692 dataOut.flagNoData = False
1677 self.__buffer = None
1693 self.__buffer = None
1678
1694
@@ -1743,7 +1759,7 class EWDriftsEstimation(Operation):
1743 dataOut.data_output = winds
1759 dataOut.data_output = winds
1744 dataOut.data_SNR = SNR1
1760 dataOut.data_SNR = SNR1
1745
1761
1746 dataOut.initUtcTime = dataOut.ltctime
1762 dataOut.utctimeInit = dataOut.utctime
1747 dataOut.outputInterval = dataOut.timeInterval
1763 dataOut.outputInterval = dataOut.timeInterval
1748 return
1764 return
1749
1765
General Comments 0
You need to be logged in to leave comments. Login now