##// END OF EJS Templates
schain
RSS
Changed name from jroIO_HDF5 to jroIO_param
Julio Valdez -
r848:85687bb4db12
parent child
Show More
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (1077 lines changed) Show them Hide them
@@ -0,0 +1,1077
1 import numpy
2 import time
3 import os
4 import h5py
5 import re
6 import datetime
7
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
13
14
15 class ParamReader(ProcessingUnit):
16 '''
17 Reads HDF5 format files
18
19 path
20
21 startDate
22
23 endDate
24
25 startTime
26
27 endTime
28 '''
29
30 ext = ".hdf5"
31
32 optchar = "D"
33
34 timezone = None
35
36 startTime = None
37
38 endTime = None
39
40 fileIndex = None
41
42 utcList = None #To select data in the utctime list
43
44 blockList = None #List to blocks to be read from the file
45
46 blocksPerFile = None #Number of blocks to be read
47
48 blockIndex = None
49
50 path = None
51
52 #List of Files
53
54 filenameList = None
55
56 datetimeList = None
57
58 #Hdf5 File
59
60 listMetaname = None
61
62 listMeta = None
63
64 listDataname = None
65
66 listData = None
67
68 listShapes = None
69
70 fp = None
71
72 #dataOut reconstruction
73
74 dataOut = None
75
76
77 def __init__(self):
78 self.dataOut = Parameters()
79 return
80
81 def setup(self, **kwargs):
82
83 path = kwargs['path']
84 startDate = kwargs['startDate']
85 endDate = kwargs['endDate']
86 startTime = kwargs['startTime']
87 endTime = kwargs['endTime']
88 walk = kwargs['walk']
89 if kwargs.has_key('ext'):
90 ext = kwargs['ext']
91 else:
92 ext = '.hdf5'
93 if kwargs.has_key('timezone'):
94 self.timezone = kwargs['timezone']
95 else:
96 self.timezone = 'lt'
97
98 print "[Reading] Searching files in offline mode ..."
99 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 startTime=startTime, endTime=endTime,
101 ext=ext, walk=walk)
102
103 if not(filenameList):
104 print "There is no files into the folder: %s"%(path)
105 sys.exit(-1)
106
107 self.fileIndex = -1
108 self.startTime = startTime
109 self.endTime = endTime
110
111 self.__readMetadata()
112
113 self.__setNextFileOffline()
114
115 return
116
117 def __searchFilesOffLine(self,
118 path,
119 startDate=None,
120 endDate=None,
121 startTime=datetime.time(0,0,0),
122 endTime=datetime.time(23,59,59),
123 ext='.hdf5',
124 walk=True):
125
126 expLabel = ''
127 self.filenameList = []
128 self.datetimeList = []
129
130 pathList = []
131
132 JRODataObj = JRODataReader()
133 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
134
135 if dateList == []:
136 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
137 datetime.datetime.combine(startDate,startTime).ctime(),
138 datetime.datetime.combine(endDate,endTime).ctime())
139
140 return None, None
141
142 if len(dateList) > 1:
143 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
144 else:
145 print "[Reading] data was found for the date %s" %(dateList[0])
146
147 filenameList = []
148 datetimeList = []
149
150 #----------------------------------------------------------------------------------
151
152 for thisPath in pathList:
153 # thisPath = pathList[pathDict[file]]
154
155 fileList = glob.glob1(thisPath, "*%s" %ext)
156 fileList.sort()
157
158 for file in fileList:
159
160 filename = os.path.join(thisPath,file)
161
162 if not isFileInDateRange(filename, startDate, endDate):
163 continue
164
165 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
166
167 if not(thisDatetime):
168 continue
169
170 filenameList.append(filename)
171 datetimeList.append(thisDatetime)
172
173 if not(filenameList):
174 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
175 return None, None
176
177 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
178 print
179
180 for i in range(len(filenameList)):
181 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
182
183 self.filenameList = filenameList
184 self.datetimeList = datetimeList
185
186 return pathList, filenameList
187
188 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
189
190 """
191 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
192
193 Inputs:
194 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
195
196 startDate : fecha inicial del rango seleccionado en formato datetime.date
197
198 endDate : fecha final del rango seleccionado en formato datetime.date
199
200 startTime : tiempo inicial del rango seleccionado en formato datetime.time
201
202 endTime : tiempo final del rango seleccionado en formato datetime.time
203
204 Return:
205 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
206 fecha especificado, de lo contrario retorna False.
207
208 Excepciones:
209 Si el archivo no existe o no puede ser abierto
210 Si la cabecera no puede ser leida.
211
212 """
213
214 try:
215 fp = h5py.File(filename,'r')
216 grp1 = fp['Data']
217
218 except IOError:
219 traceback.print_exc()
220 raise IOError, "The file %s can't be opened" %(filename)
221 #chino rata
222 #In case has utctime attribute
223 grp2 = grp1['utctime']
224 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
225 thisUtcTime = grp2.value[0]
226
227 fp.close()
228
229 if self.timezone == 'lt':
230 thisUtcTime -= 5*3600
231
232 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
233 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
234 thisDate = thisDatetime.date()
235 thisTime = thisDatetime.time()
236
237 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239
240 #General case
241 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
242 #-----------o----------------------------o-----------
243 # startTime endTime
244
245 if endTime >= startTime:
246 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
247 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
248 return thisDatetime
249 return None
250
251 #If endTime < startTime then endTime belongs to the next day
252 #<<<<<<<<<<<o o>>>>>>>>>>>
253 #-----------o----------------------------o-----------
254 # endTime startTime
255
256 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
257 return None
258
259 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
260 return None
261
262 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
263 return None
264
265 return thisDatetime
266
267 def __setNextFileOffline(self):
268
269 self.fileIndex += 1
270 idFile = self.fileIndex
271
272 if not(idFile < len(self.filenameList)):
273 print "No more Files"
274 return 0
275
276 filename = self.filenameList[idFile]
277
278 filePointer = h5py.File(filename,'r')
279
280 self.filename = filename
281
282 self.fp = filePointer
283
284 print "Setting the file: %s"%self.filename
285
286 # self.__readMetadata()
287 self.__setBlockList()
288 self.__readData()
289 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
290 # self.nRecords = self.fp['Data'].attrs['nRecords']
291 self.blockIndex = 0
292 return 1
293
294 def __setBlockList(self):
295 '''
296 Selects the data within the times defined
297
298 self.fp
299 self.startTime
300 self.endTime
301
302 self.blockList
303 self.blocksPerFile
304
305 '''
306 fp = self.fp
307 startTime = self.startTime
308 endTime = self.endTime
309
310 grp = fp['Data']
311 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
312
313 #ERROOOOR
314 if self.timezone == 'lt':
315 thisUtcTime -= 5*3600
316
317 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
318
319 thisDate = thisDatetime.date()
320 thisTime = thisDatetime.time()
321
322 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324
325 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
326
327 self.blockList = ind
328 self.blocksPerFile = len(ind)
329
330 return
331
332 def __readMetadata(self):
333 '''
334 Reads Metadata
335
336 self.pathMeta
337
338 self.listShapes
339 self.listMetaname
340 self.listMeta
341
342 '''
343
344 # grp = self.fp['Data']
345 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
346 #
347 # if pathMeta == self.pathMeta:
348 # return
349 # else:
350 # self.pathMeta = pathMeta
351 #
352 # filePointer = h5py.File(self.pathMeta,'r')
353 # groupPointer = filePointer['Metadata']
354
355 filename = self.filenameList[0]
356
357 fp = h5py.File(filename,'r')
358
359 gp = fp['Metadata']
360
361 listMetaname = []
362 listMetadata = []
363 for item in gp.items():
364 name = item[0]
365
366 if name=='array dimensions':
367 table = gp[name][:]
368 listShapes = {}
369 for shapes in table:
370 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
371 else:
372 data = gp[name].value
373 listMetaname.append(name)
374 listMetadata.append(data)
375
376 # if name=='type':
377 # self.__initDataOut(data)
378
379 self.listShapes = listShapes
380 self.listMetaname = listMetaname
381 self.listMeta = listMetadata
382
383 fp.close()
384 return
385
386 def __readData(self):
387 grp = self.fp['Data']
388 listdataname = []
389 listdata = []
390
391 for item in grp.items():
392 name = item[0]
393 listdataname.append(name)
394
395 array = self.__setDataArray(grp[name],self.listShapes[name])
396 listdata.append(array)
397
398 self.listDataname = listdataname
399 self.listData = listdata
400 return
401
402 def __setDataArray(self, dataset, shapes):
403
404 nDims = shapes[0]
405
406 nDim2 = shapes[1] #Dimension 0
407
408 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409
410 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411
412 mode = shapes[4] #Mode of storing
413
414 blockList = self.blockList
415
416 blocksPerFile = self.blocksPerFile
417
418 #Depending on what mode the data was stored
419 if mode == 0: #Divided in channels
420 arrayData = dataset.value.astype(numpy.float)[0][blockList]
421 if mode == 1: #Divided in parameter
422 strds = 'table'
423 nDatas = nDim1
424 newShapes = (blocksPerFile,nDim2,nDim0)
425 elif mode==2: #Concatenated in a table
426 strds = 'table0'
427 arrayData = dataset[strds].value
428 #Selecting part of the dataset
429 utctime = arrayData[:,0]
430 u, indices = numpy.unique(utctime, return_index=True)
431
432 if blockList.size != indices.size:
433 indMin = indices[blockList[0]]
434 if blockList[-1] + 1 >= indices.size:
435 arrayData = arrayData[indMin:,:]
436 else:
437 indMax = indices[blockList[-1] + 1]
438 arrayData = arrayData[indMin:indMax,:]
439 return arrayData
440
441 #------- One dimension ---------------
442 if nDims == 0:
443 arrayData = dataset.value.astype(numpy.float)[0][blockList]
444
445 #------- Two dimensions -----------
446 elif nDims == 2:
447 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448 newShapes = (blocksPerFile,nDim0)
449 nDatas = nDim1
450
451 for i in range(nDatas):
452 data = dataset[strds + str(i)].value
453 arrayData[:,i,:] = data[blockList,:]
454
455 #------- Three dimensions ---------
456 else:
457 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458 for i in range(nDatas):
459
460 data = dataset[strds + str(i)].value
461
462 for b in range(blockList.size):
463 arrayData[b,:,i,:] = data[:,:,blockList[b]]
464
465 return arrayData
466
467 def __setDataOut(self):
468 listMeta = self.listMeta
469 listMetaname = self.listMetaname
470 listDataname = self.listDataname
471 listData = self.listData
472 listShapes = self.listShapes
473
474 blockIndex = self.blockIndex
475 # blockList = self.blockList
476
477 for i in range(len(listMeta)):
478 setattr(self.dataOut,listMetaname[i],listMeta[i])
479
480 for j in range(len(listData)):
481 nShapes = listShapes[listDataname[j]][0]
482 mode = listShapes[listDataname[j]][4]
483 if nShapes == 1:
484 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
485 elif nShapes > 1:
486 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
487 elif mode==0:
488 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
489 #Mode Meteors
490 elif mode ==2:
491 selectedData = self.__selectDataMode2(listData[j], blockIndex)
492 setattr(self.dataOut, listDataname[j], selectedData)
493 return
494
495 def __selectDataMode2(self, data, blockIndex):
496 utctime = data[:,0]
497 aux, indices = numpy.unique(utctime, return_inverse=True)
498 selInd = numpy.where(indices == blockIndex)[0]
499 selData = data[selInd,:]
500
501 return selData
502
503 def getData(self):
504
505 # if self.flagNoMoreFiles:
506 # self.dataOut.flagNoData = True
507 # print 'Process finished'
508 # return 0
509 #
510 if self.blockIndex==self.blocksPerFile:
511 if not( self.__setNextFileOffline() ):
512 self.dataOut.flagNoData = True
513 return 0
514
515 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
516 # self.dataOut.flagNoData = True
517 # return 0
518 # self.__readData()
519 self.__setDataOut()
520 self.dataOut.flagNoData = False
521
522 self.blockIndex += 1
523
524 return
525
526 def run(self, **kwargs):
527
528 if not(self.isConfig):
529 self.setup(**kwargs)
530 # self.setObjProperties()
531 self.isConfig = True
532
533 self.getData()
534
535 return
536
537 class ParamWriter(Operation):
538 '''
539 HDF5 Writer, stores parameters data in HDF5 format files
540
541 path: path where the files will be stored
542
543 blocksPerFile: number of blocks that will be saved in per HDF5 format file
544
545 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
546
547 metadataList: list of attributes that will be stored as metadata
548
549 dataList: list of attributes that will be stores as data
550
551 '''
552
553
554 ext = ".hdf5"
555
556 optchar = "D"
557
558 metaoptchar = "M"
559
560 metaFile = None
561
562 filename = None
563
564 path = None
565
566 setFile = None
567
568 fp = None
569
570 grp = None
571
572 ds = None
573
574 firsttime = True
575
576 #Configurations
577
578 blocksPerFile = None
579
580 blockIndex = None
581
582 dataOut = None
583
584 #Data Arrays
585
586 dataList = None
587
588 metadataList = None
589
590 # arrayDim = None
591
592 dsList = None #List of dictionaries with dataset properties
593
594 tableDim = None
595
596 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
597
598 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
599
600 currentDay = None
601
602 def __init__(self):
603
604 Operation.__init__(self)
605 self.isConfig = False
606 return
607
608 def setup(self, dataOut, **kwargs):
609
610 self.path = kwargs['path']
611
612 if kwargs.has_key('blocksPerFile'):
613 self.blocksPerFile = kwargs['blocksPerFile']
614 else:
615 self.blocksPerFile = 10
616
617 self.metadataList = kwargs['metadataList']
618 self.dataList = kwargs['dataList']
619 self.dataOut = dataOut
620
621 if kwargs.has_key('mode'):
622 mode = kwargs['mode']
623
624 if type(mode) == int:
625 mode = numpy.zeros(len(self.dataList)) + mode
626 else:
627 mode = numpy.ones(len(self.dataList))
628
629 self.mode = mode
630
631 arrayDim = numpy.zeros((len(self.dataList),5))
632
633 #Table dimensions
634 dtype0 = self.dtype
635 tableList = []
636
637 #Dictionary and list of tables
638 dsList = []
639
640 for i in range(len(self.dataList)):
641 dsDict = {}
642 dataAux = getattr(self.dataOut, self.dataList[i])
643 dsDict['variable'] = self.dataList[i]
644 #--------------------- Conditionals ------------------------
645 #There is no data
646 if dataAux == None:
647 return 0
648
649 #Not array, just a number
650 #Mode 0
651 if type(dataAux)==float or type(dataAux)==int:
652 dsDict['mode'] = 0
653 dsDict['nDim'] = 0
654 arrayDim[i,0] = 0
655 dsList.append(dsDict)
656
657 #Mode 2: meteors
658 elif mode[i] == 2:
659 # dsDict['nDim'] = 0
660 dsDict['dsName'] = 'table0'
661 dsDict['mode'] = 2 # Mode meteors
662 dsDict['shape'] = dataAux.shape[-1]
663 dsDict['nDim'] = 0
664 dsDict['dsNumber'] = 1
665
666 arrayDim[i,3] = dataAux.shape[-1]
667 arrayDim[i,4] = mode[i] #Mode the data was stored
668
669 dsList.append(dsDict)
670
671 #Mode 1
672 else:
673 arrayDim0 = dataAux.shape #Data dimensions
674 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
675 arrayDim[i,4] = mode[i] #Mode the data was stored
676
677 strtable = 'table'
678 dsDict['mode'] = 1 # Mode parameters
679
680 # Three-dimension arrays
681 if len(arrayDim0) == 3:
682 arrayDim[i,1:-1] = numpy.array(arrayDim0)
683 nTables = int(arrayDim[i,2])
684 dsDict['dsNumber'] = nTables
685 dsDict['shape'] = arrayDim[i,2:4]
686 dsDict['nDim'] = 3
687
688 for j in range(nTables):
689 dsDict = dsDict.copy()
690 dsDict['dsName'] = strtable + str(j)
691 dsList.append(dsDict)
692
693 # Two-dimension arrays
694 elif len(arrayDim0) == 2:
695 arrayDim[i,2:-1] = numpy.array(arrayDim0)
696 nTables = int(arrayDim[i,2])
697 dsDict['dsNumber'] = nTables
698 dsDict['shape'] = arrayDim[i,3]
699 dsDict['nDim'] = 2
700
701 for j in range(nTables):
702 dsDict = dsDict.copy()
703 dsDict['dsName'] = strtable + str(j)
704 dsList.append(dsDict)
705
706 # One-dimension arrays
707 elif len(arrayDim0) == 1:
708 arrayDim[i,3] = arrayDim0[0]
709 dsDict['shape'] = arrayDim0[0]
710 dsDict['dsNumber'] = 1
711 dsDict['dsName'] = strtable + str(0)
712 dsDict['nDim'] = 1
713 dsList.append(dsDict)
714
715 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
716 tableList.append(table)
717
718 # self.arrayDim = arrayDim
719 self.dsList = dsList
720 self.tableDim = numpy.array(tableList, dtype = dtype0)
721 self.blockIndex = 0
722
723 timeTuple = time.localtime(dataOut.utctime)
724 self.currentDay = timeTuple.tm_yday
725 return 1
726
727 def putMetadata(self):
728
729 fp = self.createMetadataFile()
730 self.writeMetadata(fp)
731 fp.close()
732 return
733
734 def createMetadataFile(self):
735 ext = self.ext
736 path = self.path
737 setFile = self.setFile
738
739 timeTuple = time.localtime(self.dataOut.utctime)
740
741 subfolder = ''
742 fullpath = os.path.join( path, subfolder )
743
744 if not( os.path.exists(fullpath) ):
745 os.mkdir(fullpath)
746 setFile = -1 #inicializo mi contador de seteo
747
748 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
749 fullpath = os.path.join( path, subfolder )
750
751 if not( os.path.exists(fullpath) ):
752 os.mkdir(fullpath)
753 setFile = -1 #inicializo mi contador de seteo
754
755 else:
756 filesList = os.listdir( fullpath )
757 filesList = sorted( filesList, key=str.lower )
758 if len( filesList ) > 0:
759 filesList = [k for k in filesList if 'M' in k]
760 filen = filesList[-1]
761 # el filename debera tener el siguiente formato
762 # 0 1234 567 89A BCDE (hex)
763 # x YYYY DDD SSS .ext
764 if isNumber( filen[8:11] ):
765 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
766 else:
767 setFile = -1
768 else:
769 setFile = -1 #inicializo mi contador de seteo
770
771 setFile += 1
772
773 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
774 timeTuple.tm_year,
775 timeTuple.tm_yday,
776 setFile,
777 ext )
778
779 filename = os.path.join( path, subfolder, file )
780 self.metaFile = file
781 #Setting HDF5 File
782 fp = h5py.File(filename,'w')
783
784 return fp
785
786 def writeMetadata(self, fp):
787
788 grp = fp.create_group("Metadata")
789 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
790
791 for i in range(len(self.metadataList)):
792 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
793 return
794
795 def dateFlag(self):
796
797 timeTuple = time.localtime(self.dataOut.utctime)
798 dataDay = timeTuple.tm_yday
799
800 if dataDay == self.currentDay:
801 return False
802
803 self.currentDay = dataDay
804 return True
805
806 def setNextFile(self):
807
808 ext = self.ext
809 path = self.path
810 setFile = self.setFile
811 mode = self.mode
812
813 timeTuple = time.localtime(self.dataOut.utctime)
814 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
815
816 fullpath = os.path.join( path, subfolder )
817
818 if os.path.exists(fullpath):
819 filesList = os.listdir( fullpath )
820 filesList = [k for k in filesList if 'D' in k]
821 if len( filesList ) > 0:
822 filesList = sorted( filesList, key=str.lower )
823 filen = filesList[-1]
824 # el filename debera tener el siguiente formato
825 # 0 1234 567 89A BCDE (hex)
826 # x YYYY DDD SSS .ext
827 if isNumber( filen[8:11] ):
828 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
829 else:
830 setFile = -1
831 else:
832 setFile = -1 #inicializo mi contador de seteo
833 else:
834 os.mkdir(fullpath)
835 setFile = -1 #inicializo mi contador de seteo
836
837 setFile += 1
838
839 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
840 timeTuple.tm_year,
841 timeTuple.tm_yday,
842 setFile,
843 ext )
844
845 filename = os.path.join( path, subfolder, file )
846
847 #Setting HDF5 File
848 fp = h5py.File(filename,'w')
849 #write metadata
850 self.writeMetadata(fp)
851 #Write data
852 grp = fp.create_group("Data")
853 # grp.attrs['metadata'] = self.metaFile
854
855 # grp.attrs['blocksPerFile'] = 0
856 ds = []
857 data = []
858 dsList = self.dsList
859 i = 0
860 while i < len(dsList):
861 dsInfo = dsList[i]
862 #One-dimension data
863 if dsInfo['mode'] == 0:
864 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
865 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
866 ds.append(ds0)
867 data.append([])
868 i += 1
869 continue
870 # nDimsForDs.append(nDims[i])
871
872 elif dsInfo['mode'] == 2:
873 grp0 = grp.create_group(dsInfo['variable'])
874 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
875 ds.append(ds0)
876 data.append([])
877 i += 1
878 continue
879
880 elif dsInfo['mode'] == 1:
881 grp0 = grp.create_group(dsInfo['variable'])
882
883 for j in range(dsInfo['dsNumber']):
884 dsInfo = dsList[i]
885 tableName = dsInfo['dsName']
886 shape = dsInfo['shape']
887
888 if dsInfo['nDim'] == 3:
889 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
890 else:
891 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
892
893 ds.append(ds0)
894 data.append([])
895 i += 1
896 # nDimsForDs.append(nDims[i])
897
898 fp.flush()
899 fp.close()
900
901 # self.nDatas = nDatas
902 # self.nDims = nDims
903 # self.nDimsForDs = nDimsForDs
904 #Saving variables
905 print 'Writing the file: %s'%filename
906 self.filename = filename
907 # self.fp = fp
908 # self.grp = grp
909 # self.grp.attrs.modify('nRecords', 1)
910 self.ds = ds
911 self.data = data
912 # self.setFile = setFile
913 self.firsttime = True
914 self.blockIndex = 0
915 return
916
917 def putData(self):
918
919 if self.blockIndex == self.blocksPerFile or self.dateFlag():
920 self.setNextFile()
921
922 # if not self.firsttime:
923 self.readBlock()
924 self.setBlock() #Prepare data to be written
925 self.writeBlock() #Write data
926
927 return
928
929 def readBlock(self):
930
931 '''
932 data Array configured
933
934
935 self.data
936 '''
937 dsList = self.dsList
938 ds = self.ds
939 #Setting HDF5 File
940 fp = h5py.File(self.filename,'r+')
941 grp = fp["Data"]
942 ind = 0
943
944 # grp.attrs['blocksPerFile'] = 0
945 while ind < len(dsList):
946 dsInfo = dsList[ind]
947
948 if dsInfo['mode'] == 0:
949 ds0 = grp[dsInfo['variable']]
950 ds[ind] = ds0
951 ind += 1
952 else:
953
954 grp0 = grp[dsInfo['variable']]
955
956 for j in range(dsInfo['dsNumber']):
957 dsInfo = dsList[ind]
958 ds0 = grp0[dsInfo['dsName']]
959 ds[ind] = ds0
960 ind += 1
961
962 self.fp = fp
963 self.grp = grp
964 self.ds = ds
965
966 return
967
968 def setBlock(self):
969 '''
970 data Array configured
971
972
973 self.data
974 '''
975 #Creating Arrays
976 dsList = self.dsList
977 data = self.data
978 ind = 0
979
980 while ind < len(dsList):
981 dsInfo = dsList[ind]
982 dataAux = getattr(self.dataOut, dsInfo['variable'])
983
984 mode = dsInfo['mode']
985 nDim = dsInfo['nDim']
986
987 if mode == 0 or mode == 2 or nDim == 1:
988 data[ind] = dataAux
989 ind += 1
990 # elif nDim == 1:
991 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
992 # ind += 1
993 elif nDim == 2:
994 for j in range(dsInfo['dsNumber']):
995 data[ind] = dataAux[j,:]
996 ind += 1
997 elif nDim == 3:
998 for j in range(dsInfo['dsNumber']):
999 data[ind] = dataAux[:,j,:]
1000 ind += 1
1001
1002 self.data = data
1003 return
1004
1005 def writeBlock(self):
1006 '''
1007 Saves the block in the HDF5 file
1008 '''
1009 dsList = self.dsList
1010
1011 for i in range(len(self.ds)):
1012 dsInfo = dsList[i]
1013 nDim = dsInfo['nDim']
1014 mode = dsInfo['mode']
1015
1016 # First time
1017 if self.firsttime:
1018 # self.ds[i].resize(self.data[i].shape)
1019 # self.ds[i][self.blockIndex,:] = self.data[i]
1020 if type(self.data[i]) == numpy.ndarray:
1021
1022 if nDim == 3:
1023 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1024 self.ds[i].resize(self.data[i].shape)
1025 if mode == 2:
1026 self.ds[i].resize(self.data[i].shape)
1027 self.ds[i][:] = self.data[i]
1028 else:
1029
1030 # From second time
1031 # Meteors!
1032 if mode == 2:
1033 dataShape = self.data[i].shape
1034 dsShape = self.ds[i].shape
1035 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1036 self.ds[i][dsShape[0]:,:] = self.data[i]
1037 # No dimension
1038 elif mode == 0:
1039 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1040 self.ds[i][0,-1] = self.data[i]
1041 # One dimension
1042 elif nDim == 1:
1043 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1044 self.ds[i][-1,:] = self.data[i]
1045 # Two dimension
1046 elif nDim == 2:
1047 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1048 self.ds[i][self.blockIndex,:] = self.data[i]
1049 # Three dimensions
1050 elif nDim == 3:
1051 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1052 self.ds[i][:,:,-1] = self.data[i]
1053
1054 self.firsttime = False
1055 self.blockIndex += 1
1056
1057 #Close to save changes
1058 self.fp.flush()
1059 self.fp.close()
1060 return
1061
1062 def run(self, dataOut, **kwargs):
1063
1064 if not(self.isConfig):
1065 flagdata = self.setup(dataOut, **kwargs)
1066
1067 if not(flagdata):
1068 return
1069
1070 self.isConfig = True
1071 # self.putMetadata()
1072 self.setNextFile()
1073
1074 self.putData()
1075 return
1076
1077
Show file before | Show file after | Hide comments
@@ -1,14 +1,14
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JRODataIO.py 169 2012-11-19 21:57:03Z murco $
5 5 '''
6 6
7 7 from jroIO_voltage import *
8 8 from jroIO_spectra import *
9 9 from jroIO_heispectra import *
10 10 from jroIO_usrp import *
11 11
12 from jroIO_amisr import *
13 from jroIO_HDF5 import *
12 from jroIO_kamisr import *
13 from jroIO_param import *
14 14 from jroIO_hf import * No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now