##// END OF EJS Templates
schain-jc
RSS
First Draft HDF5 IO module
Julio Valdez -
r514:f095b959308c
parent child
Show More
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (656 lines changed) Show them Hide them
@@ -0,0 +1,656
1 import numpy
2 import time
3 import os
4 import h5py
5 import re
6
7 from model.data.jrodata import *
8 from model.proc.jroproc_base import ProcessingUnit, Operation
9 from model.io.jroIO_base import *
10
11
12 class HDF5Reader(ProcessingUnit):
13
14 ext = ".hdf5"
15
16 optchar = "D"
17
18 timezone = None
19
20 fileIndex = None
21
22 blockIndex = None
23
24 path = None
25
26 #Hdf5 File
27
28 fpMetadata = None
29
30 listMetaname = None
31
32 listMetadata = None
33
34 fp = None
35
36 #dataOut reconstruction
37
38
39 dataOut = None
40
41 nChannels = None #Dimension 0
42
43 nPoints = None #Dimension 1, number of Points or Parameters
44
45 nSamples = None #Dimension 2, number of samples or ranges
46
47
48 def __init__(self):
49
50 return
51
52 def setup(self,path=None,
53 startDate=None,
54 endDate=None,
55 startTime=datetime.time(0,0,0),
56 endTime=datetime.time(23,59,59),
57 walk=True,
58 timezone='ut',
59 all=0,
60 online=False,
61 ext=None):
62
63 if ext==None:
64 ext = self.ext
65 self.timezone = timezone
66 # self.all = all
67 # self.online = online
68 self.path = path
69
70
71 if not(online):
72 #Busqueda de archivos offline
73 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
74 else:
75 self.__searchFilesOnline(path, walk)
76
77 if not(self.filenameList):
78 print "There is no files into the folder: %s"%(path)
79 sys.exit(-1)
80
81 # self.__getExpParameters()
82
83 self.fileIndex = -1
84
85 self.__setNextFileOffline()
86
87 self.__readMetadata()
88
89 self.blockIndex = 0
90
91 return
92
93 def __searchFilesOffline(self,
94 path,
95 startDate,
96 endDate,
97 ext,
98 startTime=datetime.time(0,0,0),
99 endTime=datetime.time(23,59,59),
100 walk=True):
101
102 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
103 #
104 # self.__checkPath()
105 #
106 # self.__findDataForDates()
107 #
108 # self.__selectDataForTimes()
109 #
110 # for i in range(len(self.filenameList)):
111 # print "%s" %(self.filenameList[i])
112
113 pathList = []
114
115 if not walk:
116 #pathList.append(path)
117 multi_path = path.split(',')
118 for single_path in multi_path:
119 pathList.append(single_path)
120
121 else:
122 #dirList = []
123 multi_path = path.split(',')
124 for single_path in multi_path:
125 dirList = []
126 for thisPath in os.listdir(single_path):
127 if not os.path.isdir(os.path.join(single_path,thisPath)):
128 continue
129 if not isDoyFolder(thisPath):
130 continue
131
132 dirList.append(thisPath)
133
134 if not(dirList):
135 return None, None
136
137 thisDate = startDate
138
139 while(thisDate <= endDate):
140 year = thisDate.timetuple().tm_year
141 doy = thisDate.timetuple().tm_yday
142
143 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
144 if len(matchlist) == 0:
145 thisDate += datetime.timedelta(1)
146 continue
147 for match in matchlist:
148 pathList.append(os.path.join(single_path,match))
149
150 thisDate += datetime.timedelta(1)
151
152 if pathList == []:
153 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
154 return None, None
155
156 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
157
158 filenameList = []
159 datetimeList = []
160 pathDict = {}
161 filenameList_to_sort = []
162
163 for i in range(len(pathList)):
164
165 thisPath = pathList[i]
166
167 fileList = glob.glob1(thisPath, "*%s" %ext)
168 fileList.sort()
169 pathDict.setdefault(fileList[0])
170 pathDict[fileList[0]] = i
171 filenameList_to_sort.append(fileList[0])
172
173 filenameList_to_sort.sort()
174
175 for file in filenameList_to_sort:
176 thisPath = pathList[pathDict[file]]
177
178 fileList = glob.glob1(thisPath, "*%s" %ext)
179 fileList.sort()
180
181 for file in fileList:
182
183 filename = os.path.join(thisPath,file)
184 thisDatetime = self.__isFileinThisTime(filename, startTime, endTime)
185
186 if not(thisDatetime):
187 continue
188
189 filenameList.append(filename)
190 datetimeList.append(thisDatetime)
191
192 if not(filenameList):
193 print "Any file was found for the time range %s - %s" %(startTime, endTime)
194 return None, None
195
196 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
197 print
198
199 for i in range(len(filenameList)):
200 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
201
202 self.filenameList = filenameList
203 self.datetimeList = datetimeList
204
205 return pathList, filenameList
206
207 def __isFileinThisTime(self, filename, startTime, endTime):
208 """
209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
210
211 Inputs:
212 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
213
214 startTime : tiempo inicial del rango seleccionado en formato datetime.time
215
216 endTime : tiempo final del rango seleccionado en formato datetime.time
217
218 Return:
219 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
220 fecha especificado, de lo contrario retorna False.
221
222 Excepciones:
223 Si el archivo no existe o no puede ser abierto
224 Si la cabecera no puede ser leida.
225
226 """
227
228
229 try:
230 fp = fp = h5py.File(filename,'r')
231 except IOError:
232 traceback.print_exc()
233 raise IOError, "The file %s can't be opened" %(filename)
234
235 grp = fp['Data']
236 time = grp['time']
237 time0 = time[:][0]
238
239 fp.close()
240
241 thisDatetime = datetime.datetime.utcfromtimestamp(time0)
242
243 if self.timezone == 'lt':
244 thisDatetime = thisDatetime - datetime.timedelta(minutes = 300)
245
246 thisTime = thisDatetime.time()
247
248 if not ((startTime <= thisTime) and (endTime > thisTime)):
249 return None
250
251 return thisDatetime
252
253 def __checkPath(self):
254 if os.path.exists(self.path):
255 self.status = 1
256 else:
257 self.status = 0
258 print 'Path:%s does not exists'%self.path
259
260 return
261
262 def __setNextFileOffline(self):
263 idFile = self.fileIndex
264 idFile += 1
265
266 if not(idFile < len(self.filenameList)):
267 self.flagNoMoreFiles = 1
268 print "No more Files"
269 return 0
270
271 filename = self.filenameList[idFile]
272
273 filePointer = h5py.File(filename,'r')
274
275 self.flagIsNewFile = 1
276 self.fileIndex = idFile
277 self.filename = filename
278
279 self.fp = filePointer
280
281 print "Setting the file: %s"%self.filename
282
283 self.__readMetadata()
284
285 return 1
286
287 def __readMetadata(self):
288 grp = self.fp['Data']
289 self.pathMeta = os.path.join(self.path, grp.attrs['metadata'])
290 filePointer = h5py.File(self.pathMeta,'r')
291 groupPointer = filePointer['Metadata']
292
293 listMetaname = []
294 listMetadata = []
295 for item in groupPointer.items():
296 name = item[0]
297
298 if name=='data shape':
299 self.nSamples = 1
300 self.nPoints = 1
301 self.nChannels = 1
302 else:
303 data = groupPointer[name][:]
304 listMetaname.append(name)
305 listMetadata.append(data)
306
307 if name=='type':
308 self.__initDataOut(name)
309
310 filePointer.close()
311
312 self.listMetadata = listMetaname
313 self.listMetadata = listMetadata
314
315 return
316
317 def __initDataOut(self, type):
318
319 if 'type'=='Parameters':
320 self.dataOut = Parameters()
321 elif 'type'=='Spectra':
322 self.dataOut = Spectra()
323 elif 'type'=='Voltage':
324 self.dataOut = Voltage()
325 elif 'type'=='Correlation':
326 self.dataOut = Correlation()
327
328 return
329
330 def __setDataOut(self):
331 listMetadata = self.listMetadata
332 listMetaname = self.listMetaname
333 listDataname = self.listDataname
334 listData = self.listData
335
336 blockIndex = self.blockIndex
337
338 for i in range(len(listMetadata)):
339 setattr(self.dataOut,listMetaname[i],listMetadata[i])
340
341 for j in range(len(listData)):
342 setattr(self.dataOut,listDataname[j][blockIndex,:],listData[j][blockIndex,:])
343
344 return
345
346 def getData(self):
347
348 if self.flagNoMoreFiles:
349 self.dataOut.flagNoData = True
350 print 'Process finished'
351 return 0
352
353 if self.__hasNotDataInBuffer():
354 self.__setNextFile()
355
356
357 if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
358 self.dataOut.flagNoData = True
359 return 0
360
361 self.__setDataOut()
362 self.dataOut.flagNoData = False
363
364 self.blockIndex += 1
365
366 return self.dataOut.data
367
368 def run(self, **kwargs):
369
370 if not(self.isConfig):
371 self.setup(**kwargs)
372 self.setObjProperties()
373 self.isConfig = True
374
375 self.getData()
376
377 return
378
379 class HDF5Writer(Operation):
380
381 ext = ".hdf5"
382
383 optchar = "D"
384
385 metaoptchar = "M"
386
387 metaFile = None
388
389 path = None
390
391 setFile = None
392
393 fp = None
394
395 grp = None
396
397 ds = None
398
399 firsttime = True
400
401 #Configurations
402
403 blocksPerFile = None
404
405 blockIndex = None
406
407 dataOut = None
408
409 #Data Arrays
410
411 dataList = None
412
413 metadataList = None
414
415 dataDim = None
416
417 def __init__(self):
418
419 Operation.__init__(self)
420 self.isConfig = False
421 return
422
423
424 def setup(self, dataOut, **kwargs):
425
426 self.path = kwargs['path']
427
428 if kwargs.has_key('ext'):
429 self.ext = kwargs['ext']
430 else:
431 self.blocksPerFile = 10
432
433 if kwargs.has_key('blocksPerFile'):
434 self.blocksPerFile = kwargs['blocksPerFile']
435 else:
436 self.blocksPerFile = 10
437
438 self.dataOut = dataOut
439
440 self.metadataList = ['inputUnit','abscissaRange','heightRange']
441
442 self.dataList = ['data_param', 'data_error', 'data_SNR']
443
444 self.dataDim = numpy.zeros((len(self.dataList),3))
445
446 for i in range(len(self.dataList)):
447
448 dataDim = getattr(self.dataOut, self.dataList[i]).shape
449
450 if len(dataDim) == 3:
451 self.dataDim[i,:] = numpy.array(dataDim)
452 else:
453 self.dataDim[i,:-1] = numpy.array(dataDim)
454 self.dataDim[i,-1] = numpy.nan
455
456 self.blockIndex = 0
457
458 return
459
460 def putMetadata(self):
461
462 fp = self.createMetadataFile()
463 self.writeMetadata(fp)
464 fp.close()
465 return
466
467 def createMetadataFile(self):
468 ext = self.ext
469 path = self.path
470 setFile = self.setFile
471
472 timeTuple = time.localtime(self.dataOut.utctime)
473 subfolder = ''
474
475 fullpath = os.path.join( path, subfolder )
476 if not( os.path.exists(fullpath) ):
477 os.mkdir(fullpath)
478 setFile = -1 #inicializo mi contador de seteo
479 else:
480 filesList = os.listdir( fullpath )
481 if len( filesList ) > 0:
482 filesList = sorted( filesList, key=str.lower )
483 filen = filesList[-1]
484 # el filename debera tener el siguiente formato
485 # 0 1234 567 89A BCDE (hex)
486 # x YYYY DDD SSS .ext
487 if isNumber( filen[8:11] ):
488 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
489 else:
490 setFile = -1
491 else:
492 setFile = -1 #inicializo mi contador de seteo
493
494 setFile += 1
495
496 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
497 timeTuple.tm_year,
498 timeTuple.tm_yday,
499 setFile,
500 ext )
501
502 filename = os.path.join( path, subfolder, file )
503 self.metaFile = file
504 #Setting HDF5 File
505 fp = h5py.File(filename,'w')
506
507 return fp
508
509 def writeMetadata(self, fp):
510
511 grp = fp.create_group("Metadata")
512
513 for i in range(len(self.metadataList)):
514 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
515 return
516
517 def setNextFile(self):
518
519 ext = self.ext
520 path = self.path
521 setFile = self.setFile
522
523 if self.fp != None:
524 self.fp.close()
525
526 timeTuple = time.localtime(self.dataOut.utctime)
527 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
528
529 fullpath = os.path.join( path, subfolder )
530 if not( os.path.exists(fullpath) ):
531 os.mkdir(fullpath)
532 setFile = -1 #inicializo mi contador de seteo
533 else:
534 filesList = os.listdir( fullpath )
535 if len( filesList ) > 0:
536 filesList = sorted( filesList, key=str.lower )
537 filen = filesList[-1]
538 # el filename debera tener el siguiente formato
539 # 0 1234 567 89A BCDE (hex)
540 # x YYYY DDD SSS .ext
541 if isNumber( filen[8:11] ):
542 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
543 else:
544 setFile = -1
545 else:
546 setFile = -1 #inicializo mi contador de seteo
547
548 setFile += 1
549
550 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
551 timeTuple.tm_year,
552 timeTuple.tm_yday,
553 setFile,
554 ext )
555
556 filename = os.path.join( path, subfolder, file )
557
558 #Setting HDF5 File
559 fp = h5py.File(filename,'w')
560 grp = fp.create_group("Data")
561 grp.attrs['metadata'] = self.metaFile
562
563
564
565 ds = []
566 data = []
567
568 for i in range(len(self.dataList)):
569
570 grp0 = grp.create_group(self.dataList[i])
571
572 for j in range(int(self.dataDim[i,0])):
573 tableName = "channel" + str(j)
574
575 if not(numpy.isnan(self.dataDim[i,2])):
576 ds0 = grp0.create_dataset(tableName, (1,1,1) , chunks = True)
577 else:
578 ds0 = grp0.create_dataset(tableName, (1,1) , chunks = True)
579
580 ds.append(ds0)
581 data.append([])
582
583 ds0 = grp.create_dataset("time", (1,) , chunks = True)
584 ds.append(ds0)
585 data.append([])
586
587 #Saving variables
588 print 'Writing the file: %s'%filename
589 self.fp = fp
590 self.grp = grp
591 self.ds = ds
592 self.data = data
593
594 self.setFile = setFile
595 self.firsttime = True
596 self.blockIndex = 0
597 return
598
599 def putData(self):
600 self.setBlock()
601 self.writeBlock()
602
603 if self.blockIndex == self.blocksPerFile:
604 self.setNextFile()
605 return
606
607 def setBlock(self):
608
609 #Creating Arrays
610 data = self.data
611 ind = 0
612 for i in range(len(self.dataList)):
613 dataAux = getattr(self.dataOut,self.dataList[i])
614
615 for j in range(int(self.dataDim[i,0])):
616 data[ind] = dataAux[j,:]
617 if not(numpy.isnan(self.dataDim[i,2])):
618 data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
619 if not self.firsttime:
620 data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
621 else:
622 data[ind] = data[ind].reshape((1,data[ind].shape[0]))
623 if not self.firsttime:
624 data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
625 ind += 1
626
627 data[ind] = numpy.array([self.dataOut.utctime])
628 if not self.firsttime:
629 self.data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
630 self.data = data
631
632 return
633
634 def writeBlock(self):
635
636 for i in range(len(self.ds)):
637 self.ds[i].shape = self.data[i].shape
638 self.ds[i][:] = self.data[i]
639
640 self.blockIndex += 1
641
642 self.grp['blocksPerFile'] = self.blockIndex
643
644 self.firsttime = False
645 return
646
647 def run(self, dataOut, **kwargs):
648 if not(self.isConfig):
649 self.setup(dataOut, **kwargs)
650 self.isConfig = True
651 self.putMetadata()
652 self.setNextFile()
653
654 self.putData()
655 return
656
Show file before | Show file after | Hide comments
@@ -0,0 +1,65
1 # DIAS 19 Y 20 FEB 2014
2 # Comprobacion de Resultados DBS con SA
3
4 import os, sys
5
6 path = os.path.split(os.getcwd())[0]
7 sys.path.append(path)
8
9 from controller import *
10
11 desc = "DBS Experiment Test"
12 filename = "DBStest.xml"
13
14 controllerObj = Project()
15
16 controllerObj.setup(id = '191', name='test01', description=desc)
17
18 #Experimentos
19
20 path = "/home/soporte/Data/drifts/HDF5"
21 pathFigure = '/home/soporte/workspace/Graficos/drifts/prueba'
22 pathFile = '/home/soporte/Data/drifts/HDF5'
23
24 xmin = 0
25 xmax = 24
26 #------------------------------------------------------------------------------------------------
27 readUnitConfObj = controllerObj.addReadUnit(datatype='HDF5Reader',
28 path=path,
29 startDate='2012/09/06',
30 endDate='2012/09/06',
31 startTime='00:00:00',
32 endTime='23:59:59',
33 timezone='lt',
34 walk=1)
35
36 procUnitConfObj0 = controllerObj.addProcUnit(datatype='ParametersProc', inputId=readUnitConfObj.getId())
37 #--------------------------------------------------------------------------------------------------
38
39 opObj11 = procUnitConfObj0.addOperation(name='EWDriftsEstimation', optype='other')
40 opObj11.addParameter(name='zenith', value='-3.80208,3.10658', format='floatlist')
41 opObj11.addParameter(name='zenithCorrection', value='0.183201', format='float')
42
43 opObj23 = procUnitConfObj0.addOperation(name='EWDriftsPlot', optype='other')
44 opObj23.addParameter(name='id', value='4', format='int')
45 opObj23.addParameter(name='wintitle', value='EW Drifts', format='str')
46 opObj23.addParameter(name='save', value='1', format='bool')
47 opObj23.addParameter(name='figpath', value = pathFigure, format='str')
48 opObj23.addParameter(name='zminZonal', value='-150', format='int')
49 opObj23.addParameter(name='zmaxZonal', value='150', format='int')
50 opObj23.addParameter(name='zminVertical', value='-30', format='float')
51 opObj23.addParameter(name='zmaxVertical', value='30', format='float')
52 opObj23.addParameter(name='SNR_1', value='1', format='bool')
53 opObj23.addParameter(name='SNRmax', value='5', format='int')
54 # opObj23.addParameter(name='SNRthresh', value='-50', format='float')
55 opObj23.addParameter(name='xmin', value=xmin, format='float')
56 opObj23.addParameter(name='xmax', value=xmax, format='float')
57 #--------------------------------------------------------------------------------------------------
58 print "Escribiendo el archivo XML"
59 controllerObj.writeXml(filename)
60 print "Leyendo el archivo XML"
61 controllerObj.readXml(filename)
62
63 controllerObj.createObjects()
64 controllerObj.connectObjects()
65 controllerObj.run() No newline at end of file
Show file before | Show file after | Hide comments
@@ -9,6 +9,8 from xml.dom import minidom
9 import datetime
9 import datetime
10 from model import *
10 from model import *
11
11
12 import ast
13
12 def prettify(elem):
14 def prettify(elem):
13 """Return a pretty-printed XML string for the Element.
15 """Return a pretty-printed XML string for the Element.
14 """
16 """
@@ -89,6 +91,14 class ParameterConf():
89
91
90 return pairList
92 return pairList
91
93
94 if self.format == 'multiList':
95 """
96 Example:
97 value = (0,1,2),(3,4,5)
98 """
99 multiList = ast.literal_eval(value)
100 return multiList
101
92 func = eval(self.format)
102 func = eval(self.format)
93
103
94 self.__value = func(value)
104 self.__value = func(value)
Show file before | Show file after | Hide comments
@@ -933,23 +933,23 class Parameters(JROData):
933
933
934 data_pre = None #Data Pre Parametrization
934 data_pre = None #Data Pre Parametrization
935
935
936 data_SNR = None #Signal to Noise Ratio
937
936 heightRange = None #Heights
938 heightRange = None #Heights
937
939
938 abscissaRange = None #Abscissa, can be velocities, lags or time
940 abscissaRange = None #Abscissa, can be velocities, lags or time
939
941
940 noise = None #Noise Potency
942 noise = None #Noise Potency
941
943
942 SNR = None #Signal to Noise Ratio
943
944 initUtcTime = None #Initial UTC time
944 initUtcTime = None #Initial UTC time
945
945
946 paramInterval = None #Time interval to calculate Parameters in seconds
946 paramInterval = None #Time interval to calculate Parameters in seconds
947
947
948 #Fitting
948 #Fitting
949
949
950 constants = None
950 data_error = None #Error of the estimation
951
951
952 error = None
952 constants = None
953
953
954 library = None
954 library = None
955
955
Show file before | Show file after | Hide comments
@@ -460,9 +460,9 class WindProfilerPlot(Figure):
460 nplotsw = nplots
460 nplotsw = nplots
461
461
462 #If there is a SNR function defined
462 #If there is a SNR function defined
463 if dataOut.SNR != None:
463 if dataOut.data_SNR != None:
464 nplots += 1
464 nplots += 1
465 SNR = dataOut.SNR
465 SNR = dataOut.data_SNR
466 SNRavg = numpy.average(SNR, axis=0)
466 SNRavg = numpy.average(SNR, axis=0)
467
467
468 SNRdB = 10*numpy.log10(SNR)
468 SNRdB = 10*numpy.log10(SNR)
@@ -503,7 +503,7 class WindProfilerPlot(Figure):
503 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
503 if zmax_ver == None: zmax_ver = numpy.nanmax(abs(z[2,:]))
504 if zmin_ver == None: zmin_ver = -zmax_ver
504 if zmin_ver == None: zmin_ver = -zmax_ver
505
505
506 if dataOut.SNR != None:
506 if dataOut.data_SNR != None:
507 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
507 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
508 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
508 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
509
509
@@ -539,7 +539,7 class WindProfilerPlot(Figure):
539 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
539 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
540 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="RdBu_r" )
540 ticksize=9, cblabel=strCb[i], cbsize="1%", colormap="RdBu_r" )
541
541
542 if dataOut.SNR != None:
542 if dataOut.data_SNR != None:
543 i += 1
543 i += 1
544 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
544 title = "Signal Noise Ratio (SNR): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
545 axes = self.axesList[i*self.__nsubplots]
545 axes = self.axesList[i*self.__nsubplots]
@@ -719,7 +719,7 class ParametersPlot(Figure):
719 if zmin == None: zmin = numpy.nanmin(zRange)
719 if zmin == None: zmin = numpy.nanmin(zRange)
720 if zmax == None: zmax = numpy.nanmax(zRange)
720 if zmax == None: zmax = numpy.nanmax(zRange)
721
721
722 if dataOut.SNR != None:
722 if dataOut.data_SNR != None:
723 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
723 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
724 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
724 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
725
725
@@ -1061,9 +1061,9 class EWDriftsPlot(Figure):
1061 nplotsw = nplots
1061 nplotsw = nplots
1062
1062
1063 #If there is a SNR function defined
1063 #If there is a SNR function defined
1064 if dataOut.SNR != None:
1064 if dataOut.data_SNR != None:
1065 nplots += 1
1065 nplots += 1
1066 SNR = dataOut.SNR
1066 SNR = dataOut.data_SNR
1067
1067
1068 if SNR_1:
1068 if SNR_1:
1069 SNR += 1
1069 SNR += 1
@@ -1104,7 +1104,7 class EWDriftsPlot(Figure):
1104 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1104 if zmaxVertical == None: zmaxVertical = numpy.nanmax(abs(z[1,:]))
1105 if zminVertical == None: zminVertical = -zmaxVertical
1105 if zminVertical == None: zminVertical = -zmaxVertical
1106
1106
1107 if dataOut.SNR != None:
1107 if dataOut.data_SNR != None:
1108 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1108 if SNRmin == None: SNRmin = numpy.nanmin(SNRavgdB)
1109 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1109 if SNRmax == None: SNRmax = numpy.nanmax(SNRavgdB)
1110
1110
@@ -1139,7 +1139,7 class EWDriftsPlot(Figure):
1139 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1139 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
1140 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1140 ticksize=9, cblabel=strCb, cbsize="1%", colormap="RdBu_r")
1141
1141
1142 if dataOut.SNR != None:
1142 if dataOut.data_SNR != None:
1143 i += 1
1143 i += 1
1144 if SNR_1:
1144 if SNR_1:
1145 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1145 title = "Signal Noise Ratio + 1 (SNR+1): %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
Show file before | Show file after | Hide comments
@@ -1,4 +1,5
1 from jroIO_voltage import *
1 from jroIO_voltage import *
2 from jroIO_spectra import *
2 from jroIO_spectra import *
3 from jroIO_heispectra import *
3 from jroIO_heispectra import *
4 from jroIO_amisr import * No newline at end of file
4 from jroIO_amisr import *
5 from jroIO_HDF5 import * No newline at end of file
Show file before | Show file after | Hide comments
@@ -115,7 +115,7 class ParametersProc(ProcessingUnit):
115 self.dataOut.abscissaRange = self.dataIn.getLagTRange(1)
115 self.dataOut.abscissaRange = self.dataIn.getLagTRange(1)
116 self.dataOut.noise = self.dataIn.noise
116 self.dataOut.noise = self.dataIn.noise
117 self.dataOut.normFactor = self.dataIn.normFactor
117 self.dataOut.normFactor = self.dataIn.normFactor
118 self.dataOut.SNR = self.dataIn.SNR
118 self.dataOut.data_SNR = self.dataIn.SNR
119 self.dataOut.groupList = self.dataIn.pairsList
119 self.dataOut.groupList = self.dataIn.pairsList
120 self.dataOut.flagNoData = False
120 self.dataOut.flagNoData = False
121
121
@@ -138,7 +138,7 class ParametersProc(ProcessingUnit):
138
138
139 Affected:
139 Affected:
140 self.dataOut.data_param
140 self.dataOut.data_param
141 self.dataOut.SNR
141 self.dataOut.data_SNR
142
142
143 '''
143 '''
144 data = self.dataOut.data_pre
144 data = self.dataOut.data_pre
@@ -155,7 +155,7 class ParametersProc(ProcessingUnit):
155 data_param[ind,:,:] = self.__calculateMoments(data[ind,:,:], absc, noise[ind])
155 data_param[ind,:,:] = self.__calculateMoments(data[ind,:,:], absc, noise[ind])
156
156
157 self.dataOut.data_param = data_param[:,1:,:]
157 self.dataOut.data_param = data_param[:,1:,:]
158 self.dataOut.SNR = data_param[:,0]
158 self.dataOut.data_SNR = data_param[:,0]
159 return
159 return
160
160
161 def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
161 def __calculateMoments(self, oldspec, oldfreq, n0, nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):
@@ -241,7 +241,7 class ParametersProc(ProcessingUnit):
241 self.dataOut.abscissaRange
241 self.dataOut.abscissaRange
242 self.dataOut.noise
242 self.dataOut.noise
243 self.dataOut.normFactor
243 self.dataOut.normFactor
244 self.dataOut.SNR
244 self.dataOut.data_SNR
245 self.dataOut.groupList
245 self.dataOut.groupList
246 self.dataOut.nChannels
246 self.dataOut.nChannels
247
247
@@ -254,7 +254,7 class ParametersProc(ProcessingUnit):
254 nHeights = self.dataOut.nHeights
254 nHeights = self.dataOut.nHeights
255 absc = self.dataOut.abscissaRange[:-1]
255 absc = self.dataOut.abscissaRange[:-1]
256 noise = self.dataOut.noise
256 noise = self.dataOut.noise
257 SNR = self.dataOut.SNR
257 SNR = self.dataOut.data_SNR
258 pairsList = self.dataOut.groupList
258 pairsList = self.dataOut.groupList
259 nChannels = self.dataOut.nChannels
259 nChannels = self.dataOut.nChannels
260 pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels)
260 pairsAutoCorr, pairsCrossCorr = self.__getPairsAutoCorr(pairsList, nChannels)
@@ -1155,7 +1155,7 class ParametersProc(ProcessingUnit):
1155 listChannels = groupArray.reshape((groupArray.size))
1155 listChannels = groupArray.reshape((groupArray.size))
1156 listChannels.sort()
1156 listChannels.sort()
1157 noise = self.dataIn.getNoise()
1157 noise = self.dataIn.getNoise()
1158 self.dataOut.SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
1158 self.dataOut.data_SNR = self.__getSNR(self.dataIn.data_spc[listChannels,:,:], noise[listChannels])
1159
1159
1160 for i in range(nGroups):
1160 for i in range(nGroups):
1161 coord = groupArray[i,:]
1161 coord = groupArray[i,:]
@@ -1201,22 +1201,27 class ParametersProc(ProcessingUnit):
1201
1201
1202 #Initial values
1202 #Initial values
1203 data_spc = self.dataIn.data_spc[coord,:,h]
1203 data_spc = self.dataIn.data_spc[coord,:,h]
1204 p0 = self.dataOut.library.initialValuesFunction(data_spc, constants)
1204 p0 = numpy.array(self.dataOut.library.initialValuesFunction(data_spc, constants))
1205
1205
1206 #Least Squares
1206 try:
1207 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
1207 #Least Squares
1208 # minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants))
1208 minp,covp,infodict,mesg,ier = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants),full_output=True)
1209 #Chi square error
1209 # minp,covp = optimize.leastsq(self.__residFunction,p0,args=(dp,LT,constants))
1210 error0 = numpy.sum(infodict['fvec']**2)/(2*N)
1210 #Chi square error
1211 # error0 = 0
1211 error0 = numpy.sum(infodict['fvec']**2)/(2*N)
1212 #Error with Jacobian
1212 #Error with Jacobian
1213 error1 = self.dataOut.library.errorFunction(minp,constants,LT)
1213 error1 = self.dataOut.library.errorFunction(minp,constants,LT)
1214 except:
1215 minp = p0*numpy.nan
1216 error0 = numpy.nan
1217 error1 = p0*numpy.nan
1218
1214 #Save
1219 #Save
1215 if self.dataOut.data_param == None:
1220 if self.dataOut.data_param == None:
1216 self.dataOut.data_param = numpy.zeros((nGroups, minp.size, nHeights))*numpy.nan
1221 self.dataOut.data_param = numpy.zeros((nGroups, p0.size, nHeights))*numpy.nan
1217 self.dataOut.error = numpy.zeros((nGroups, error1.size + 1, nHeights))*numpy.nan
1222 self.dataOut.data_error = numpy.zeros((nGroups, p0.size + 1, nHeights))*numpy.nan
1218
1223
1219 self.dataOut.error[i,:,h] = numpy.hstack((error0,error1))
1224 self.dataOut.data_error[i,:,h] = numpy.hstack((error0,error1))
1220 self.dataOut.data_param[i,:,h] = minp
1225 self.dataOut.data_param[i,:,h] = minp
1221 return
1226 return
1222
1227
@@ -1569,7 +1574,7 class WindProfiler(Operation):
1569 absc = dataOut.abscissaRange[:-1]
1574 absc = dataOut.abscissaRange[:-1]
1570 noise = dataOut.noise
1575 noise = dataOut.noise
1571 heightRange = dataOut.getHeiRange()
1576 heightRange = dataOut.getHeiRange()
1572 SNR = dataOut.SNR
1577 SNR = dataOut.data_SNR
1573
1578
1574 if technique == 'DBS':
1579 if technique == 'DBS':
1575
1580
@@ -1597,7 +1602,7 class WindProfiler(Operation):
1597 theta_y = theta_y[arrayChannel]
1602 theta_y = theta_y[arrayChannel]
1598
1603
1599 velRadial0 = param[:,1,:] #Radial velocity
1604 velRadial0 = param[:,1,:] #Radial velocity
1600 dataOut.data_output, dataOut.heightRange, dataOut.SNR = self.techniqueDBS(velRadial0, theta_x, theta_y, azimuth, correctFactor, horizontalOnly, heightRange, SNR) #DBS Function
1605 dataOut.data_output, dataOut.heightRange, dataOut.data_SNR = self.techniqueDBS(velRadial0, theta_x, theta_y, azimuth, correctFactor, horizontalOnly, heightRange, SNR) #DBS Function
1601
1606
1602 elif technique == 'SA':
1607 elif technique == 'SA':
1603
1608
@@ -1715,7 +1720,7 class EWDriftsEstimation(Operation):
1715 def run(self, dataOut, zenith, zenithCorrection):
1720 def run(self, dataOut, zenith, zenithCorrection):
1716 heiRang = dataOut.heightList
1721 heiRang = dataOut.heightList
1717 velRadial = dataOut.data_param[:,3,:]
1722 velRadial = dataOut.data_param[:,3,:]
1718 SNR = dataOut.SNR
1723 SNR = dataOut.data_SNR
1719
1724
1720 zenith = numpy.array(zenith)
1725 zenith = numpy.array(zenith)
1721 zenith -= zenithCorrection
1726 zenith -= zenithCorrection
@@ -1736,7 +1741,7 class EWDriftsEstimation(Operation):
1736
1741
1737 dataOut.heightList = heiRang1
1742 dataOut.heightList = heiRang1
1738 dataOut.data_output = winds
1743 dataOut.data_output = winds
1739 dataOut.SNR = SNR1
1744 dataOut.data_SNR = SNR1
1740
1745
1741 dataOut.initUtcTime = dataOut.ltctime
1746 dataOut.initUtcTime = dataOut.ltctime
1742 dataOut.outputInterval = dataOut.timeInterval
1747 dataOut.outputInterval = dataOut.timeInterval
Show file before | Show file after | Hide comments
@@ -23,14 +23,15 pathFigure = '/home/propietario/workspace/Graficos/drifts'
23
23
24 path = "/home/soporte/Data/drifts"
24 path = "/home/soporte/Data/drifts"
25 pathFigure = '/home/soporte/workspace/Graficos/drifts/prueba'
25 pathFigure = '/home/soporte/workspace/Graficos/drifts/prueba'
26 pathFile = '/home/soporte/Data/drifts/HDF5'
26
27
27 xmin = 11.75
28 xmin = 0
28 xmax = 14.75
29 xmax = 24
29 #------------------------------------------------------------------------------------------------
30 #------------------------------------------------------------------------------------------------
30 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
31 readUnitConfObj = controllerObj.addReadUnit(datatype='VoltageReader',
31 path=path,
32 path=path,
32 startDate='2012/01/01',
33 startDate='2012/09/06',
33 endDate='2012/12/31',
34 endDate='2012/09/06',
34 startTime='00:00:00',
35 startTime='00:00:00',
35 endTime='23:59:59',
36 endTime='23:59:59',
36 online=0,
37 online=0,
@@ -106,6 +107,10 opObj11.addParameter(name='fit', value='1', format='int')#1--True/include fit
106 opObj11.addParameter(name='save', value='1', format='bool')
107 opObj11.addParameter(name='save', value='1', format='bool')
107 opObj11.addParameter(name='figpath', value = pathFigure, format='str')
108 opObj11.addParameter(name='figpath', value = pathFigure, format='str')
108
109
110 opObj12 = procUnitConfObj2.addOperation(name='HDF5Writer', optype='other')
111 opObj12.addParameter(name='path', value=pathFile)
112 opObj12.addParameter(name='blocksPerFile', value='3', format='int')
113
109 opObj11 = procUnitConfObj2.addOperation(name='EWDriftsEstimation', optype='other')
114 opObj11 = procUnitConfObj2.addOperation(name='EWDriftsEstimation', optype='other')
110 opObj11.addParameter(name='zenith', value='-3.80208,3.10658', format='floatlist')
115 opObj11.addParameter(name='zenith', value='-3.80208,3.10658', format='floatlist')
111 opObj11.addParameter(name='zenithCorrection', value='0.183201', format='float')
116 opObj11.addParameter(name='zenithCorrection', value='0.183201', format='float')
Show file before | Show file after | Hide comments
@@ -109,7 +109,7 opObj21.addParameter(name='figpath', value=pathFigure, format='str')
109 opObj21.addParameter(name='zmin', value='5', format='int')
109 opObj21.addParameter(name='zmin', value='5', format='int')
110 opObj21.addParameter(name='zmax', value='90', format='int')
110 opObj21.addParameter(name='zmax', value='90', format='int')
111
111
112 opObj21 = procUnitConfObj2.addOperation(name='RadialVelocityPlot', optype='other')
112 opObj21 = procUnitConfObj2.addOperation(name='ParametersPlot', optype='other')
113 opObj21.addParameter(name='id', value='5', format='int')
113 opObj21.addParameter(name='id', value='5', format='int')
114 opObj21.addParameter(name='wintitle', value='Radial Velocity Plot', format='str')
114 opObj21.addParameter(name='wintitle', value='Radial Velocity Plot', format='str')
115 opObj21.addParameter(name='save', value='1', format='bool')
115 opObj21.addParameter(name='save', value='1', format='bool')
@@ -119,6 +119,20 opObj21.addParameter(name='SNRmax', value='60', format='int')
119 opObj21.addParameter(name='SNRthresh', value='0', format='float')
119 opObj21.addParameter(name='SNRthresh', value='0', format='float')
120 opObj21.addParameter(name='xmin', value=xmin, format='float')
120 opObj21.addParameter(name='xmin', value=xmin, format='float')
121 opObj21.addParameter(name='xmax', value=xmax, format='float')
121 opObj21.addParameter(name='xmax', value=xmax, format='float')
122
123 opObj21 = procUnitConfObj2.addOperation(name='ParametersPlot', optype='other')
124 opObj21.addParameter(name='id', value='6', format='int')
125 opObj21.addParameter(name='wintitle', value='Spectral width Plot', format='str')
126 opObj21.addParameter(name='save', value='1', format='bool')
127 opObj21.addParameter(name='figpath', value=pathFigure, format='str')
128 opObj21.addParameter(name='SNRmin', value='-10', format='int')
129 opObj21.addParameter(name='SNRmax', value='60', format='int')
130 opObj21.addParameter(name='SNRthresh', value='0', format='float')
131 opObj21.addParameter(name='xmin', value=xmin, format='float')
132 opObj21.addParameter(name='xmax', value=xmax, format='float')
133 opObj21.addParameter(name='zmin', value=0, format='float')
134 opObj21.addParameter(name='paramIndex', value=2, format='int')
135 opObj21.addParameter(name='onlyPositive', value=1, format='bool')
122
136
123 opObj22 = procUnitConfObj2.addOperation(name='WindProfiler', optype='other')
137 opObj22 = procUnitConfObj2.addOperation(name='WindProfiler', optype='other')
124 opObj22.addParameter(name='technique', value='DBS', format='str')
138 opObj22.addParameter(name='technique', value='DBS', format='str')
Show file before | Show file after | Hide comments
@@ -20,7 +20,7 controllerObj.setup(id = '191', name='test01', description=desc)
20
20
21 #2014050 19 Feb 2014
21 #2014050 19 Feb 2014
22 path = '/home/soporte/Data/MST/SA/d2014050'
22 path = '/home/soporte/Data/MST/SA/d2014050'
23 pathFigure = '/home/soporte/workspace/Graficos/SA/new1/'
23 pathFigure = '/home/soporte/workspace/Graficos/SA/prueba1/'
24 xmin = '15.5'
24 xmin = '15.5'
25 xmax = '24'
25 xmax = '24'
26 startTime = '15:30:00'
26 startTime = '15:30:00'
General Comments 0
You need to be logged in to leave comments. Login now