##// END OF EJS Templates
schain
RSS
-Functional HDF5 file writer unit...
Julio Valdez -
r543:5c87f3286f40
parent child
Show More
Show file before | Show file after | Hide comments
@@ -111,8 +111,8 class MomentsPlot(Figure):
111 111 channelIndexList.append(dataOut.channelList.index(channel))
112 112
113 113 factor = dataOut.normFactor
114 x = dataOut.abscissaRange
115 y = dataOut.heightRange
114 x = dataOut.abscissaList
115 y = dataOut.heightList
116 116
117 117 z = dataOut.data_pre[channelIndexList,:,:]/factor
118 118 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
@@ -452,8 +452,8 class WindProfilerPlot(Figure):
452 452 tmax = None
453 453
454 454 x = dataOut.getTimeRange1()
455 # y = dataOut.heightRange
456 y = dataOut.heightRange
455 # y = dataOut.heightList
456 y = dataOut.heightList
457 457
458 458 z = dataOut.data_output.copy()
459 459 nplots = z.shape[0] #Number of wind dimensions estimated
@@ -649,8 +649,8 class ParametersPlot(Figure):
649 649
650 650 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=False,
651 651 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,timerange=None,
652 SNRmin = None, SNRmax = None, SNRthresh = None, paramIndex = None, onlyPositive = False,
653 zlabel = "", parameterName = "",
652 parameterIndex = None, onlyPositive = False,
653 zlabel = "", parameterName = "", parameterObject = "data_param",
654 654 save=False, figpath='', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
655 655 server=None, folder=None, username=None, password=None,
656 656 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
@@ -671,27 +671,25 class ParametersPlot(Figure):
671 671 zmax : None
672 672 """
673 673
674 data_param = getattr(dataOut, parameterObject)
675
674 676 if channelList == None:
675 channelIndexList = dataOut.channelIndexList
677 channelIndexList = numpy.arange(data_param.shape[0])
676 678 else:
677 channelIndexList = []
678 for channel in channelList:
679 if channel not in dataOut.channelList:
680 raise ValueError, "Channel %d is not in dataOut.channelList"
681 channelIndexList.append(dataOut.channelList.index(channel))
682
679 channelIndexList = numpy.array(channelIndexList)
680
683 681 if timerange != None:
684 682 self.timerange = timerange
685 683
686 684 #tmin = None
687 685 #tmax = None
688 if paramIndex == None:
689 paramIndex = 1
686 if parameterIndex == None:
687 parameterIndex = 1
690 688 x = dataOut.getTimeRange1()
691 y = dataOut.heightRange
692 z = dataOut.data_param[channelIndexList,paramIndex,:].copy()
689 y = dataOut.heightList
690 z = data_param[channelIndexList,parameterIndex,:].copy()
693 691
694 zRange = dataOut.abscissaRange
692 zRange = dataOut.abscissaList
695 693 nplots = z.shape[0] #Number of wind dimensions estimated
696 694 # thisDatetime = dataOut.datatime
697 695 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
@@ -718,10 +716,6 class ParametersPlot(Figure):
718 716 if ymax == None: ymax = numpy.nanmax(y)
719 717 if zmin == None: zmin = numpy.nanmin(zRange)
720 718 if zmax == None: zmax = numpy.nanmax(zRange)
721
722 if dataOut.data_SNR != None:
723 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
724 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
725 719
726 720 self.FTP_WEI = ftp_wei
727 721 self.EXP_CODE = exp_code
@@ -862,7 +856,7 class SpectralFittingPlot(Figure):
862 856 cutHeight = dataOut.heightList[heightindex]
863 857
864 858 factor = dataOut.normFactor
865 x = dataOut.abscissaRange[:-1]
859 x = dataOut.abscissaList[:-1]
866 860 #y = dataOut.getHeiRange()
867 861
868 862 z = dataOut.data_pre[:,:,heightindex]/factor
@@ -1037,15 +1031,6 class EWDriftsPlot(Figure):
1037 1031 zmax : None
1038 1032 """
1039 1033
1040 if channelList == None:
1041 channelIndexList = dataOut.channelIndexList
1042 else:
1043 channelIndexList = []
1044 for channel in channelList:
1045 if channel not in dataOut.channelList:
1046 raise ValueError, "Channel %d is not in dataOut.channelList"
1047 channelIndexList.append(dataOut.channelList.index(channel))
1048
1049 1034 if timerange != None:
1050 1035 self.timerange = timerange
1051 1036
@@ -1053,7 +1038,7 class EWDriftsPlot(Figure):
1053 1038 tmax = None
1054 1039
1055 1040 x = dataOut.getTimeRange1()
1056 # y = dataOut.heightRange
1041 # y = dataOut.heightList
1057 1042 y = dataOut.heightList
1058 1043
1059 1044 z = dataOut.data_output
@@ -1081,7 +1066,7 class EWDriftsPlot(Figure):
1081 1066
1082 1067 showprofile = False
1083 1068 # thisDatetime = dataOut.datatime
1084 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[1])
1069 thisDatetime = datetime.datetime.utcfromtimestamp(x[1])
1085 1070 title = wintitle + " EW Drifts"
1086 1071 xlabel = ""
1087 1072 ylabel = "Height (Km)"
Show file before | Show file after | Hide comments
@@ -17,38 +17,59 class HDF5Reader(ProcessingUnit):
17 17
18 18 timezone = None
19 19
20 secStart = None
21
22 secEnd = None
23
20 24 fileIndex = None
21 25
22 26 blockIndex = None
23 27
28 blocksPerFile = None
29
24 30 path = None
25 31
32 #List of Files
33
34 filenameList = None
35
36 datetimeList = None
37
26 38 #Hdf5 File
27 39
28 40 fpMetadata = None
29 41
42 pathMeta = None
43
30 44 listMetaname = None
31 45
32 listMetadata = None
46 listMeta = None
33 47
34 fp = None
48 listDataname = None
35 49
36 #dataOut reconstruction
50 listData = None
37 51
52 listShapes = None
38 53
39 dataOut = None
54 fp = None
40 55
41 nChannels = None #Dimension 0
56 #dataOut reconstruction
42 57
43 nPoints = None #Dimension 1, number of Points or Parameters
58 dataOut = None
44 59
45 nSamples = None #Dimension 2, number of samples or ranges
60 nRecords = None
46 61
47 62
48 63 def __init__(self):
49
64 self.dataOut = self.__createObjByDefault()
50 65 return
66
67 def __createObjByDefault(self):
51 68
69 dataObj = Parameters()
70
71 return dataObj
72
52 73 def setup(self,path=None,
53 74 startDate=None,
54 75 endDate=None,
@@ -66,11 +87,18 class HDF5Reader(ProcessingUnit):
66 87 # self.all = all
67 88 # self.online = online
68 89 self.path = path
69
90
91 startDateTime = datetime.datetime.combine(startDate,startTime)
92 endDateTime = datetime.datetime.combine(endDate,endTime)
93 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
94 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
95
96 self.secStart = secStart
97 self.secEnd = secEnd
70 98
71 99 if not(online):
72 100 #Busqueda de archivos offline
73 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
101 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
74 102 else:
75 103 self.__searchFilesOnline(path, walk)
76 104
@@ -97,6 +125,8 class HDF5Reader(ProcessingUnit):
97 125 ext,
98 126 startTime=datetime.time(0,0,0),
99 127 endTime=datetime.time(23,59,59),
128 secStart = 0,
129 secEnd = numpy.inf,
100 130 walk=True):
101 131
102 132 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
@@ -181,7 +211,7 class HDF5Reader(ProcessingUnit):
181 211 for file in fileList:
182 212
183 213 filename = os.path.join(thisPath,file)
184 thisDatetime = self.__isFileinThisTime(filename, startTime, endTime)
214 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
185 215
186 216 if not(thisDatetime):
187 217 continue
@@ -204,7 +234,7 class HDF5Reader(ProcessingUnit):
204 234
205 235 return pathList, filenameList
206 236
207 def __isFileinThisTime(self, filename, startTime, endTime):
237 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
208 238 """
209 239 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210 240
@@ -224,8 +254,7 class HDF5Reader(ProcessingUnit):
224 254 Si la cabecera no puede ser leida.
225 255
226 256 """
227
228
257
229 258 try:
230 259 fp = fp = h5py.File(filename,'r')
231 260 except IOError:
@@ -233,21 +262,20 class HDF5Reader(ProcessingUnit):
233 262 raise IOError, "The file %s can't be opened" %(filename)
234 263
235 264 grp = fp['Data']
236 time = grp['time']
237 time0 = time[:][0]
265 timeAux = grp['time']
266 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
238 267
239 268 fp.close()
240 269
241 thisDatetime = datetime.datetime.utcfromtimestamp(time0)
242
243 270 if self.timezone == 'lt':
244 thisDatetime = thisDatetime - datetime.timedelta(minutes = 300)
245
246 thisTime = thisDatetime.time()
247
248 if not ((startTime <= thisTime) and (endTime > thisTime)):
271 time0 -= 5*3600
272
273 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
274
275 if not (numpy.any(boolTimer)):
249 276 return None
250 277
278 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
251 279 return thisDatetime
252 280
253 281 def __checkPath(self):
@@ -264,7 +292,6 class HDF5Reader(ProcessingUnit):
264 292 idFile += 1
265 293
266 294 if not(idFile < len(self.filenameList)):
267 self.flagNoMoreFiles = 1
268 295 print "No more Files"
269 296 return 0
270 297
@@ -281,12 +308,57 class HDF5Reader(ProcessingUnit):
281 308 print "Setting the file: %s"%self.filename
282 309
283 310 self.__readMetadata()
284
311 self.__setBlockList()
312 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
313 self.nRecords = self.fp['Data'].attrs['nRecords']
314 self.blockIndex = 0
285 315 return 1
286 316
317 def __setBlockList(self):
318 '''
319 self.fp
320 self.startDateTime
321 self.endDateTime
322
323 self.blockList
324 self.blocksPerFile
325
326 '''
327 filePointer = self.fp
328 secStart = self.secStart
329 secEnd = self.secEnd
330
331 grp = filePointer['Data']
332 timeVector = grp['time'].value.astype(numpy.float)[0]
333
334 if self.timezone == 'lt':
335 timeVector -= 5*3600
336
337 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
338
339 self.blockList = ind
340 self.blocksPerFile = len(ind)
341
342 return
343
287 344 def __readMetadata(self):
345 '''
346 self.pathMeta
347
348 self.listShapes
349 self.listMetaname
350 self.listMeta
351
352 '''
353
288 354 grp = self.fp['Data']
289 self.pathMeta = os.path.join(self.path, grp.attrs['metadata'])
355 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
356
357 if pathMeta == self.pathMeta:
358 return
359 else:
360 self.pathMeta = pathMeta
361
290 362 filePointer = h5py.File(self.pathMeta,'r')
291 363 groupPointer = filePointer['Metadata']
292 364
@@ -295,81 +367,172 class HDF5Reader(ProcessingUnit):
295 367 for item in groupPointer.items():
296 368 name = item[0]
297 369
298 if name=='data shape':
299 self.nSamples = 1
300 self.nPoints = 1
301 self.nChannels = 1
370 if name=='array dimensions':
371 table = groupPointer[name][:]
372 listShapes = {}
373 for shapes in table:
374 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
302 375 else:
303 data = groupPointer[name][:]
376 data = groupPointer[name].value
304 377 listMetaname.append(name)
305 378 listMetadata.append(data)
306 379
307 380 if name=='type':
308 self.__initDataOut(name)
381 self.__initDataOut(data)
309 382
310 383 filePointer.close()
311 384
312 self.listMetadata = listMetaname
313 self.listMetadata = listMetadata
385 self.listShapes = listShapes
386 self.listMetaname = listMetaname
387 self.listMeta = listMetadata
314 388
315 389 return
316 390
391 def __readData(self):
392 grp = self.fp['Data']
393 listdataname = []
394 listdata = []
395
396 for item in grp.items():
397 name = item[0]
398
399 if name == 'time':
400 listdataname.append('utctime')
401 timeAux = grp[name].value.astype(numpy.float)[0]
402 listdata.append(timeAux)
403 continue
404
405 listdataname.append(name)
406 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
407 listdata.append(array)
408
409 self.listDataname = listdataname
410 self.listData = listdata
411 return
412
413 def __setDataArray(self, nRecords, dataset, shapes):
414
415 nChannels = shapes[0] #Dimension 0
416
417 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
418
419 nSamples = shapes[2] #Dimension 2, number of samples or ranges
420
421 mode = shapes[3]
422
423 # if nPoints>1:
424 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
425 # else:
426 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
427 #
428 # chn = 'channel'
429 #
430 # for i in range(nChannels):
431 #
432 # data = dataset[chn + str(i)].value
433 #
434 # if nPoints>1:
435 # data = numpy.rollaxis(data,2)
436 #
437 # arrayData[:,i,:] = data
438
439 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
440 doSqueeze = False
441 if mode == 0:
442 strds = 'channel'
443 nDatas = nChannels
444 newShapes = (nRecords,nPoints,nSamples)
445 if nPoints == 1:
446 doSqueeze = True
447 axisSqueeze = 2
448 else:
449 strds = 'param'
450 nDatas = nPoints
451 newShapes = (nRecords,nChannels,nSamples)
452 if nChannels == 1:
453 doSqueeze = True
454 axisSqueeze = 1
455
456 for i in range(nDatas):
457
458 data = dataset[strds + str(i)].value
459 data = data.reshape(newShapes)
460
461 if mode == 0:
462 arrayData[:,i,:,:] = data
463 else:
464 arrayData[:,:,i,:] = data
465
466 if doSqueeze:
467 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
468
469 return arrayData
470
317 471 def __initDataOut(self, type):
318 472
319 if 'type'=='Parameters':
320 self.dataOut = Parameters()
321 elif 'type'=='Spectra':
322 self.dataOut = Spectra()
323 elif 'type'=='Voltage':
324 self.dataOut = Voltage()
325 elif 'type'=='Correlation':
326 self.dataOut = Correlation()
473 # if type =='Parameters':
474 # self.dataOut = Parameters()
475 # elif type =='Spectra':
476 # self.dataOut = Spectra()
477 # elif type =='Voltage':
478 # self.dataOut = Voltage()
479 # elif type =='Correlation':
480 # self.dataOut = Correlation()
327 481
328 482 return
329 483
330 484 def __setDataOut(self):
331 listMetadata = self.listMetadata
485 listMeta = self.listMeta
332 486 listMetaname = self.listMetaname
333 487 listDataname = self.listDataname
334 488 listData = self.listData
335 489
336 490 blockIndex = self.blockIndex
491 blockList = self.blockList
337 492
338 for i in range(len(listMetadata)):
339 setattr(self.dataOut,listMetaname[i],listMetadata[i])
493 for i in range(len(listMeta)):
494 setattr(self.dataOut,listMetaname[i],listMeta[i])
340 495
341 496 for j in range(len(listData)):
342 setattr(self.dataOut,listDataname[j][blockIndex,:],listData[j][blockIndex,:])
497 if listDataname[j]=='utctime':
498 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
499 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
500 continue
501
502 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
343 503
344 return
504 return self.dataOut.data_param
345 505
346 506 def getData(self):
347 507
348 if self.flagNoMoreFiles:
349 self.dataOut.flagNoData = True
350 print 'Process finished'
351 return 0
352
353 if self.__hasNotDataInBuffer():
354 self.__setNextFile()
355
356
357 if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
358 self.dataOut.flagNoData = True
359 return 0
508 # if self.flagNoMoreFiles:
509 # self.dataOut.flagNoData = True
510 # print 'Process finished'
511 # return 0
512 #
513 if self.blockIndex==self.blocksPerFile:
514 if not( self.__setNextFileOffline() ):
515 self.dataOut.flagNoData = True
516 return 0
517
518 #
519 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
520 # self.dataOut.flagNoData = True
521 # return 0
360 522
523 self.__readData()
361 524 self.__setDataOut()
362 525 self.dataOut.flagNoData = False
363 526
364 527 self.blockIndex += 1
365 528
366 return self.dataOut.data
529 return
367 530
368 531 def run(self, **kwargs):
369 532
370 533 if not(self.isConfig):
371 534 self.setup(**kwargs)
372 self.setObjProperties()
535 # self.setObjProperties()
373 536 self.isConfig = True
374 537
375 538 self.getData()
@@ -412,11 +575,19 class HDF5Writer(Operation):
412 575
413 576 metadataList = None
414 577
415 dataDim = None
578 arrayDim = None
416 579
417 580 tableDim = None
418 581
419 dtype = [('arrayName', 'S10'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i')]
582 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
583
584 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
585
586 mode = None
587
588 nDatas = None #Number of datasets to be stored per array
589
590 nDims = None #Number Dimensions in each dataset
420 591
421 592 def __init__(self):
422 593
@@ -431,23 +602,31 class HDF5Writer(Operation):
431 602
432 603 if kwargs.has_key('ext'):
433 604 self.ext = kwargs['ext']
434 else:
435 self.blocksPerFile = 10
436
605
437 606 if kwargs.has_key('blocksPerFile'):
438 607 self.blocksPerFile = kwargs['blocksPerFile']
439 608 else:
440 609 self.blocksPerFile = 10
441 610
611 self.metadataList = kwargs['metadataList']
612
613 self.dataList = kwargs['dataList']
614
442 615 self.dataOut = dataOut
443 616
444 self.metadataList = ['type','inputUnit','abscissaRange','heightRange']
445
446 self.dataList = ['data_param', 'data_error', 'data_SNR']
617 if kwargs.has_key('mode'):
618 mode = kwargs['mode']
619
620 if type(mode) == int:
621 mode = numpy.zeros(len(self.dataList)) + mode
622 else:
623 mode = numpy.zeros(len(self.dataList))
624
625 self.mode = mode
447 626
448 self.dataDim = numpy.zeros((len(self.dataList),3))
627 arrayDim = numpy.zeros((len(self.dataList),5))
449 628
450 #Data types
629 #Table dimensions
451 630
452 631 dtype0 = self.dtype
453 632
@@ -455,18 +634,29 class HDF5Writer(Operation):
455 634
456 635 for i in range(len(self.dataList)):
457 636
458 dataDim = getattr(self.dataOut, self.dataList[i]).shape
637 dataAux = getattr(self.dataOut, self.dataList[i])
459 638
460 if len(dataDim) == 3:
461 self.dataDim[i,:] = numpy.array(dataDim)
639 if type(dataAux)==float or type(dataAux)==int:
640 arrayDim[i,0] = 1
462 641 else:
463 self.dataDim[i,0] = numpy.array(dataDim)[0]
464 self.dataDim[i,2] = numpy.array(dataDim)[1]
465 self.dataDim[i,1] = 1
642 arrayDim0 = dataAux.shape
643 arrayDim[i,0] = len(arrayDim0)
644 arrayDim[i,4] = mode[i]
466 645
467 table = numpy.array((self.dataList[i],) + tuple(self.dataDim[i,:]),dtype = dtype0)
646 if len(arrayDim0) == 3:
647 arrayDim[i,1:-1] = numpy.array(arrayDim0)
648 elif len(arrayDim0) == 2:
649 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
650 elif len(arrayDim0) == 1:
651 arrayDim[i,3] = arrayDim0
652 elif len(arrayDim0) == 0:
653 arrayDim[i,0] = 1
654 arrayDim[i,3] = 1
655
656 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
468 657 tableList.append(table)
469
658
659 self.arrayDim = arrayDim
470 660 self.tableDim = numpy.array(tableList, dtype = dtype0)
471 661 self.blockIndex = 0
472 662
@@ -535,6 +725,7 class HDF5Writer(Operation):
535 725 ext = self.ext
536 726 path = self.path
537 727 setFile = self.setFile
728 mode = self.mode
538 729
539 730 if self.fp != None:
540 731 self.fp.close()
@@ -576,29 +767,48 class HDF5Writer(Operation):
576 767 grp = fp.create_group("Data")
577 768 grp.attrs['metadata'] = self.metaFile
578 769
579 grp['blocksPerFile'] = 0
770 # grp.attrs['blocksPerFile'] = 0
580 771
581 772 ds = []
582 773 data = []
583 774
584 for i in range(len(self.dataList)):
775 nDatas = numpy.zeros(len(self.dataList))
776 nDims = self.arrayDim[:,0]
777
778 for i in range(len(self.dataList)):
585 779
586 grp0 = grp.create_group(self.dataList[i])
587
588 for j in range(int(self.dataDim[i,0])):
589 tableName = "channel" + str(j)
590
591 if not(self.dataDim[i,1] == 1):
592 ds0 = grp0.create_dataset(tableName, (1,1,1) , chunks = True)
593 else:
594 ds0 = grp0.create_dataset(tableName, (1,1) , chunks = True)
595
780 if nDims[i]==1:
781 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,None) , chunks = True, dtype='S20')
596 782 ds.append(ds0)
597 783 data.append([])
598
599 ds0 = grp.create_dataset("time", (1,) , chunks = True)
600 ds.append(ds0)
601 data.append([])
784
785 else:
786
787 if mode[i]==0:
788 strMode = "channel"
789 nDatas[i] = self.arrayDim[i,1]
790 else:
791 strMode = "param"
792 nDatas[i] = self.arrayDim[i,2]
793
794 if nDims[i]==2:
795 nDatas[i] = self.arrayDim[i,2]
796
797 grp0 = grp.create_group(self.dataList[i])
798
799 for j in range(int(nDatas[i])):
800 tableName = strMode + str(j)
801
802 if nDims[i] == 3:
803 ds0 = grp0.create_dataset(tableName, (1,1,1) , maxshape=(None,None,None), chunks=True)
804 else:
805 ds0 = grp0.create_dataset(tableName, (1,1) , maxshape=(None,None), chunks=True)
806
807 ds.append(ds0)
808 data.append([])
809
810 self.nDatas = nDatas
811 self.nDims = nDims
602 812
603 813 #Saving variables
604 814 print 'Writing the file: %s'%filename
@@ -624,31 +834,46 class HDF5Writer(Operation):
624 834 '''
625 835 data Array configured
626 836
837
838 self.data
627 839 '''
628 840 #Creating Arrays
629 841 data = self.data
842 nDatas = self.nDatas
843 nDims = self.nDims
844 mode = self.mode
630 845 ind = 0
846
631 847 for i in range(len(self.dataList)):
632 848 dataAux = getattr(self.dataOut,self.dataList[i])
633 849
634 for j in range(int(self.dataDim[i,0])):
635 data[ind] = dataAux[j,:]
636
637 if not(self.dataDim[i,1] == 1):
638 data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
639 if not self.firsttime:
640 data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
641 else:
642 data[ind] = data[ind].reshape((1,data[ind].shape[0]))
643 if not self.firsttime:
644 data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
850 if nDims[i] == 1:
851 data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
852 if not self.firsttime:
853 data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
645 854 ind += 1
646 855
647 data[ind] = numpy.array([self.dataOut.utctime])
648 if not self.firsttime:
649 self.data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
856 else:
857 for j in range(int(nDatas[i])):
858 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
859 data[ind] = dataAux[j,:]
860 else:
861 data[ind] = dataAux[:,j,:]
862
863 if nDims[i] == 3:
864 data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
865
866 if not self.firsttime:
867 data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
868
869 else:
870 data[ind] = data[ind].reshape((1,data[ind].shape[0]))
871
872 if not self.firsttime:
873 data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
874 ind += 1
875
650 876 self.data = data
651
652 877 return
653 878
654 879 def writeBlock(self):
@@ -656,12 +881,12 class HDF5Writer(Operation):
656 881 Saves the block in the HDF5 file
657 882 '''
658 883 for i in range(len(self.ds)):
659 self.ds[i].shape = self.data[i].shape
884 self.ds[i].resize(self.data[i].shape)
660 885 self.ds[i][:] = self.data[i]
661 886
662 887 self.blockIndex += 1
663 888
664 self.grp.attrs.modify('blocksPerFile', self.blockIndex)
889 self.grp.attrs.modify('nRecords', self.blockIndex)
665 890
666 891 self.firsttime = False
667 892 return
Show file before | Show file after | Hide comments
@@ -22,7 +22,7 class ParametersProc(ProcessingUnit):
22 22 def __init__(self):
23 23 ProcessingUnit.__init__(self)
24 24
25 self.objectDict = {}
25 # self.objectDict = {}
26 26 self.buffer = None
27 27 self.firstdatatime = None
28 28 self.profIndex = 0
@@ -57,7 +57,7 class ParametersProc(ProcessingUnit):
57 57 self.dataOut.ippSeconds = self.dataIn.ippSeconds
58 58 # self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
59 59 self.dataOut.timeInterval = self.dataIn.timeInterval
60 self.dataOut.heightRange = self.dataIn.getHeiRange()
60 self.dataOut.heightList = self.dataIn.getHeiRange()
61 61 self.dataOut.frequency = self.dataIn.frequency
62 62
63 63 def run(self, nSeconds = None, nProfiles = None):
@@ -100,7 +100,7 class ParametersProc(ProcessingUnit):
100 100
101 101 if self.dataIn.type == "Spectra":
102 102 self.dataOut.data_pre = self.dataIn.data_spc.copy()
103 self.dataOut.abscissaRange = self.dataIn.getVelRange(1)
103 self.dataOut.abscissaList = self.dataIn.getVelRange(1)
104 104 self.dataOut.noise = self.dataIn.getNoise()
105 105 self.dataOut.normFactor = self.dataIn.normFactor
106 106 self.dataOut.flagNoData = False
@@ -112,17 +112,24 class ParametersProc(ProcessingUnit):
112 112 indR = numpy.where(lagRRange == 0)[0][0]
113 113
114 114 self.dataOut.data_pre = self.dataIn.data_corr.copy()[:,:,indR,:]
115 self.dataOut.abscissaRange = self.dataIn.getLagTRange(1)
115 self.dataOut.abscissaList = self.dataIn.getLagTRange(1)
116 116 self.dataOut.noise = self.dataIn.noise
117 117 self.dataOut.normFactor = self.dataIn.normFactor
118 118 self.dataOut.data_SNR = self.dataIn.SNR
119 119 self.dataOut.groupList = self.dataIn.pairsList
120 120 self.dataOut.flagNoData = False
121
122 #---------------------- Correlation Data ---------------------------
123
124 if self.dataIn.type == "Parameters":
125 self.dataOut.copy(self.dataIn)
126 self.dataOut.flagNoData = False
121 127
128 return True
122 129
123 130 self.__updateObjFromInput()
124 131 self.firstdatatime = None
125 self.dataOut.initUtcTime = self.dataIn.ltctime
132 self.dataOut.utctimeInit = self.dataIn.utctime
126 133 self.dataOut.outputInterval = self.dataIn.timeInterval
127 134
128 135 #------------------- Get Moments ----------------------------------
@@ -133,7 +140,7 class ParametersProc(ProcessingUnit):
133 140 Input:
134 141 channelList : simple channel list to select e.g. [2,3,7]
135 142 self.dataOut.data_pre
136 self.dataOut.abscissaRange
143 self.dataOut.abscissaList
137 144 self.dataOut.noise
138 145
139 146 Affected:
@@ -142,7 +149,7 class ParametersProc(ProcessingUnit):
142 149
143 150 '''
144 151 data = self.dataOut.data_pre
145 absc = self.dataOut.abscissaRange[:-1]
152 absc = self.dataOut.abscissaList[:-1]
146 153 noise = self.dataOut.noise
147 154
148 155 data_param = numpy.zeros((data.shape[0], 4, data.shape[2]))
@@ -238,7 +245,7 class ParametersProc(ProcessingUnit):
238 245
239 246 Input:
240 247 self.dataOut.data_pre
241 self.dataOut.abscissaRange
248 self.dataOut.abscissaList
242 249 self.dataOut.noise
243 250 self.dataOut.normFactor
244 251 self.dataOut.data_SNR
@@ -252,7 +259,7 class ParametersProc(ProcessingUnit):
252 259 data = self.dataOut.data_pre
253 260 normFactor = self.dataOut.normFactor
254 261 nHeights = self.dataOut.nHeights
255 absc = self.dataOut.abscissaRange[:-1]
262 absc = self.dataOut.abscissaList[:-1]
256 263 noise = self.dataOut.noise
257 264 SNR = self.dataOut.data_SNR
258 265 pairsList = self.dataOut.groupList
@@ -1201,7 +1208,11 class ParametersProc(ProcessingUnit):
1201 1208
1202 1209 #Initial values
1203 1210 data_spc = self.dataIn.data_spc[coord,:,h]
1204 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants))
1211
1212 if (h>0)and(error1[3]<5):
1213 p0 = self.dataOut.data_param[i,:,h-1]
1214 else:
1215 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants, i))
1205 1216
1206 1217 try:
1207 1218 #Least Squares
@@ -1570,10 +1581,10 class WindProfiler(Operation):
1570 1581 def run(self, dataOut, technique, **kwargs):
1571 1582
1572 1583 param = dataOut.data_param
1573 if dataOut.abscissaRange != None:
1574 absc = dataOut.abscissaRange[:-1]
1584 if dataOut.abscissaList != None:
1585 absc = dataOut.abscissaList[:-1]
1575 1586 noise = dataOut.noise
1576 heightRange = dataOut.getHeiRange()
1587 heightList = dataOut.getHeiRange()
1577 1588 SNR = dataOut.data_SNR
1578 1589
1579 1590 if technique == 'DBS':
@@ -1602,7 +1613,9 class WindProfiler(Operation):
1602 1613 theta_y = theta_y[arrayChannel]
1603 1614
1604 1615 velRadial0 = param[:,1,:] #Radial velocity
1605 dataOut.data_output, dataOut.heightRange, dataOut.data_SNR = self.techniqueDBS(velRadial0, theta_x, theta_y, azimuth, correctFactor, horizontalOnly, heightRange, SNR) #DBS Function
1616 dataOut.data_output, dataOut.heightList, dataOut.data_SNR = self.techniqueDBS(velRadial0, theta_x, theta_y, azimuth, correctFactor, horizontalOnly, heightList, SNR) #DBS Function
1617 dataOut.utctimeInit = dataOut.utctime
1618 dataOut.outputInterval = dataOut.timeInterval
1606 1619
1607 1620 elif technique == 'SA':
1608 1621
@@ -1627,7 +1640,7 class WindProfiler(Operation):
1627 1640 nChannels = dataOut.nChannels
1628 1641
1629 1642 dataOut.data_output = self.techniqueSA(pairs, pairsList, nChannels, tau, azimuth, _lambda, position_x, position_y, absc, correctFactor)
1630 dataOut.initUtcTime = dataOut.ltctime
1643 dataOut.utctimeInit = dataOut.utctime
1631 1644 dataOut.outputInterval = dataOut.timeInterval
1632 1645
1633 1646 elif technique == 'Meteors':
@@ -1656,7 +1669,9 class WindProfiler(Operation):
1656 1669 if self.__isConfig == False:
1657 1670 # self.__initime = dataOut.datatime.replace(minute = 0, second = 0, microsecond = 03)
1658 1671 #Get Initial LTC time
1659 self.__initime = (dataOut.datatime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1672 self.__initime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
1673 self.__initime = (self.__initime.replace(minute = 0, second = 0, microsecond = 0) - datetime.datetime(1970, 1, 1)).total_seconds()
1674
1660 1675 self.__isConfig = True
1661 1676
1662 1677 if self.__buffer == None:
@@ -1666,13 +1681,14 class WindProfiler(Operation):
1666 1681 else:
1667 1682 self.__buffer = numpy.vstack((self.__buffer, dataOut.data_param))
1668 1683
1669 self.__checkTime(dataOut.ltctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1684 self.__checkTime(dataOut.utctime, dataOut.paramInterval, dataOut.outputInterval) #Check if the buffer is ready
1670 1685
1671 1686 if self.__dataReady:
1672 dataOut.initUtcTime = self.__initime
1673 self.__initime = self.__initime + dataOut.outputInterval #to erase time offset
1687 dataOut.utctimeInit = self.__initime
1688
1689 self.__initime += dataOut.outputInterval #to erase time offset
1674 1690
1675 dataOut.data_output, dataOut.heightRange = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
1691 dataOut.data_output, dataOut.heightList = self.techniqueMeteors(self.__buffer, meteorThresh, hmin, hmax)
1676 1692 dataOut.flagNoData = False
1677 1693 self.__buffer = None
1678 1694
@@ -1743,7 +1759,7 class EWDriftsEstimation(Operation):
1743 1759 dataOut.data_output = winds
1744 1760 dataOut.data_SNR = SNR1
1745 1761
1746 dataOut.initUtcTime = dataOut.ltctime
1762 dataOut.utctimeInit = dataOut.utctime
1747 1763 dataOut.outputInterval = dataOut.timeInterval
1748 1764 return
1749 1765
General Comments 0
You need to be logged in to leave comments. Login now