##// END OF EJS Templates
schain
RSS
Bug fix in Reading Unit
Julio Valdez -
r821:375dfaaf810f
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1067 +1,1067
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class HDF5Reader(ProcessingUnit):
15 class HDF5Reader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self):
77 def __init__(self):
78 self.dataOut = Parameters()
78 self.dataOut = Parameters()
79 return
79 return
80
80
81 def setup(self, **kwargs):
81 def setup(self, **kwargs):
82
82
83 path = kwargs['path']
83 path = kwargs['path']
84 startDate = kwargs['startDate']
84 startDate = kwargs['startDate']
85 endDate = kwargs['endDate']
85 endDate = kwargs['endDate']
86 startTime = kwargs['startTime']
86 startTime = kwargs['startTime']
87 endTime = kwargs['endTime']
87 endTime = kwargs['endTime']
88 walk = kwargs['walk']
88 walk = kwargs['walk']
89 if kwargs.has_key('ext'):
89 if kwargs.has_key('ext'):
90 ext = kwargs['ext']
90 ext = kwargs['ext']
91 else:
91 else:
92 ext = '.hdf5'
92 ext = '.hdf5'
93
93
94 print "[Reading] Searching files in offline mode ..."
94 print "[Reading] Searching files in offline mode ..."
95 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
95 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
96 startTime=startTime, endTime=endTime,
96 startTime=startTime, endTime=endTime,
97 ext=ext, walk=walk)
97 ext=ext, walk=walk)
98
98
99 if not(filenameList):
99 if not(filenameList):
100 print "There is no files into the folder: %s"%(path)
100 print "There is no files into the folder: %s"%(path)
101 sys.exit(-1)
101 sys.exit(-1)
102
102
103 self.fileIndex = -1
103 self.fileIndex = -1
104 self.startTime = startTime
104 self.startTime = startTime
105 self.endTime = endTime
105 self.endTime = endTime
106
106
107 self.__readMetadata()
107 self.__readMetadata()
108
108
109 self.__setNextFileOffline()
109 self.__setNextFileOffline()
110
110
111 return
111 return
112
112
113 def __searchFilesOffLine(self,
113 def __searchFilesOffLine(self,
114 path,
114 path,
115 startDate=None,
115 startDate=None,
116 endDate=None,
116 endDate=None,
117 startTime=datetime.time(0,0,0),
117 startTime=datetime.time(0,0,0),
118 endTime=datetime.time(23,59,59),
118 endTime=datetime.time(23,59,59),
119 ext='.hdf5',
119 ext='.hdf5',
120 walk=True):
120 walk=True):
121
121
122 expLabel = ''
122 expLabel = ''
123 self.filenameList = []
123 self.filenameList = []
124 self.datetimeList = []
124 self.datetimeList = []
125
125
126 pathList = []
126 pathList = []
127
127
128 JRODataObj = JRODataReader()
128 JRODataObj = JRODataReader()
129 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
129 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
130
130
131 if dateList == []:
131 if dateList == []:
132 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
132 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
133 datetime.datetime.combine(startDate,startTime).ctime(),
133 datetime.datetime.combine(startDate,startTime).ctime(),
134 datetime.datetime.combine(endDate,endTime).ctime())
134 datetime.datetime.combine(endDate,endTime).ctime())
135
135
136 return None, None
136 return None, None
137
137
138 if len(dateList) > 1:
138 if len(dateList) > 1:
139 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
139 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
140 else:
140 else:
141 print "[Reading] data was found for the date %s" %(dateList[0])
141 print "[Reading] data was found for the date %s" %(dateList[0])
142
142
143 filenameList = []
143 filenameList = []
144 datetimeList = []
144 datetimeList = []
145
145
146 #----------------------------------------------------------------------------------
146 #----------------------------------------------------------------------------------
147
147
148 for thisPath in pathList:
148 for thisPath in pathList:
149 # thisPath = pathList[pathDict[file]]
149 # thisPath = pathList[pathDict[file]]
150
150
151 fileList = glob.glob1(thisPath, "*%s" %ext)
151 fileList = glob.glob1(thisPath, "*%s" %ext)
152 fileList.sort()
152 fileList.sort()
153
153
154 for file in fileList:
154 for file in fileList:
155
155
156 filename = os.path.join(thisPath,file)
156 filename = os.path.join(thisPath,file)
157
157
158 if not isFileInDateRange(filename, startDate, endDate):
158 if not isFileInDateRange(filename, startDate, endDate):
159 continue
159 continue
160
160
161 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
161 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
162
162
163 if not(thisDatetime):
163 if not(thisDatetime):
164 continue
164 continue
165
165
166 filenameList.append(filename)
166 filenameList.append(filename)
167 datetimeList.append(thisDatetime)
167 datetimeList.append(thisDatetime)
168
168
169 if not(filenameList):
169 if not(filenameList):
170 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
170 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
171 return None, None
171 return None, None
172
172
173 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
173 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
174 print
174 print
175
175
176 for i in range(len(filenameList)):
176 for i in range(len(filenameList)):
177 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
177 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
178
178
179 self.filenameList = filenameList
179 self.filenameList = filenameList
180 self.datetimeList = datetimeList
180 self.datetimeList = datetimeList
181
181
182 return pathList, filenameList
182 return pathList, filenameList
183
183
184 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
184 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
185
185
186 """
186 """
187 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
187 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
188
188
189 Inputs:
189 Inputs:
190 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
190 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
191
191
192 startDate : fecha inicial del rango seleccionado en formato datetime.date
192 startDate : fecha inicial del rango seleccionado en formato datetime.date
193
193
194 endDate : fecha final del rango seleccionado en formato datetime.date
194 endDate : fecha final del rango seleccionado en formato datetime.date
195
195
196 startTime : tiempo inicial del rango seleccionado en formato datetime.time
196 startTime : tiempo inicial del rango seleccionado en formato datetime.time
197
197
198 endTime : tiempo final del rango seleccionado en formato datetime.time
198 endTime : tiempo final del rango seleccionado en formato datetime.time
199
199
200 Return:
200 Return:
201 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
201 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
202 fecha especificado, de lo contrario retorna False.
202 fecha especificado, de lo contrario retorna False.
203
203
204 Excepciones:
204 Excepciones:
205 Si el archivo no existe o no puede ser abierto
205 Si el archivo no existe o no puede ser abierto
206 Si la cabecera no puede ser leida.
206 Si la cabecera no puede ser leida.
207
207
208 """
208 """
209
209
210 try:
210 try:
211 fp = h5py.File(filename,'r')
211 fp = h5py.File(filename,'r')
212 grp1 = fp['Data']
212 grp1 = fp['Data']
213
213
214 except IOError:
214 except IOError:
215 traceback.print_exc()
215 traceback.print_exc()
216 raise IOError, "The file %s can't be opened" %(filename)
216 raise IOError, "The file %s can't be opened" %(filename)
217 #chino rata
217 #chino rata
218 #In case has utctime attribute
218 #In case has utctime attribute
219 grp2 = grp1['utctime']
219 grp2 = grp1['utctime']
220 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
220 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
221 thisUtcTime = grp2.value[0]
221 thisUtcTime = grp2.value[0]
222
222
223 fp.close()
223 fp.close()
224
224
225 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
225 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
226 thisDate = thisDatetime.date()
226 thisDate = thisDatetime.date()
227 thisTime = thisDatetime.time()
227 thisTime = thisDatetime.time()
228
228
229 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
229 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
230 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
230 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
231
231
232 #General case
232 #General case
233 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
233 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
234 #-----------o----------------------------o-----------
234 #-----------o----------------------------o-----------
235 # startTime endTime
235 # startTime endTime
236
236
237 if endTime >= startTime:
237 if endTime >= startTime:
238 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
238 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
239 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
239 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
240 return thisDatetime
240 return thisDatetime
241 return None
241 return None
242
242
243 #If endTime < startTime then endTime belongs to the next day
243 #If endTime < startTime then endTime belongs to the next day
244 #<<<<<<<<<<<o o>>>>>>>>>>>
244 #<<<<<<<<<<<o o>>>>>>>>>>>
245 #-----------o----------------------------o-----------
245 #-----------o----------------------------o-----------
246 # endTime startTime
246 # endTime startTime
247
247
248 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
248 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
249 return None
249 return None
250
250
251 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
251 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
252 return None
252 return None
253
253
254 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
254 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
255 return None
255 return None
256
256
257 return thisDatetime
257 return thisDatetime
258
258
259 def __setNextFileOffline(self):
259 def __setNextFileOffline(self):
260
260
261 self.fileIndex += 1
261 self.fileIndex += 1
262 idFile = self.fileIndex
262 idFile = self.fileIndex
263
263
264 if not(idFile < len(self.filenameList)):
264 if not(idFile < len(self.filenameList)):
265 print "No more Files"
265 print "No more Files"
266 return 0
266 return 0
267
267
268 filename = self.filenameList[idFile]
268 filename = self.filenameList[idFile]
269
269
270 filePointer = h5py.File(filename,'r')
270 filePointer = h5py.File(filename,'r')
271
271
272 self.filename = filename
272 self.filename = filename
273
273
274 self.fp = filePointer
274 self.fp = filePointer
275
275
276 print "Setting the file: %s"%self.filename
276 print "Setting the file: %s"%self.filename
277
277
278 # self.__readMetadata()
278 # self.__readMetadata()
279 self.__setBlockList()
279 self.__setBlockList()
280 self.__readData()
280 self.__readData()
281 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
281 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
282 # self.nRecords = self.fp['Data'].attrs['nRecords']
282 # self.nRecords = self.fp['Data'].attrs['nRecords']
283 self.blockIndex = 0
283 self.blockIndex = 0
284 return 1
284 return 1
285
285
286 def __setBlockList(self):
286 def __setBlockList(self):
287 '''
287 '''
288 Selects the data within the times defined
288 Selects the data within the times defined
289
289
290 self.fp
290 self.fp
291 self.startTime
291 self.startTime
292 self.endTime
292 self.endTime
293
293
294 self.blockList
294 self.blockList
295 self.blocksPerFile
295 self.blocksPerFile
296
296
297 '''
297 '''
298 fp = self.fp
298 fp = self.fp
299 startTime = self.startTime
299 startTime = self.startTime
300 endTime = self.endTime
300 endTime = self.endTime
301
301
302 grp = fp['Data']
302 grp = fp['Data']
303 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
303 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
304
304
305 if self.timezone == 'lt':
305 if self.timezone == 'lt':
306 thisUtcTime -= 5*3600
306 thisUtcTime -= 5*3600
307
307
308 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
308 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
309 thisDate = thisDatetime.date()
309 thisDate = thisDatetime.date()
310 thisTime = thisDatetime.time()
310 thisTime = thisDatetime.time()
311
311
312 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
312 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
313 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
313 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
314
314
315 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
315 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
316
316
317 self.blockList = ind
317 self.blockList = ind
318 self.blocksPerFile = len(ind)
318 self.blocksPerFile = len(ind)
319
319
320 return
320 return
321
321
322 def __readMetadata(self):
322 def __readMetadata(self):
323 '''
323 '''
324 Reads Metadata
324 Reads Metadata
325
325
326 self.pathMeta
326 self.pathMeta
327
327
328 self.listShapes
328 self.listShapes
329 self.listMetaname
329 self.listMetaname
330 self.listMeta
330 self.listMeta
331
331
332 '''
332 '''
333
333
334 # grp = self.fp['Data']
334 # grp = self.fp['Data']
335 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
335 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
336 #
336 #
337 # if pathMeta == self.pathMeta:
337 # if pathMeta == self.pathMeta:
338 # return
338 # return
339 # else:
339 # else:
340 # self.pathMeta = pathMeta
340 # self.pathMeta = pathMeta
341 #
341 #
342 # filePointer = h5py.File(self.pathMeta,'r')
342 # filePointer = h5py.File(self.pathMeta,'r')
343 # groupPointer = filePointer['Metadata']
343 # groupPointer = filePointer['Metadata']
344
344
345 filename = self.filenameList[0]
345 filename = self.filenameList[0]
346
346
347 fp = h5py.File(filename,'r')
347 fp = h5py.File(filename,'r')
348
348
349 gp = fp['Metadata']
349 gp = fp['Metadata']
350
350
351 listMetaname = []
351 listMetaname = []
352 listMetadata = []
352 listMetadata = []
353 for item in gp.items():
353 for item in gp.items():
354 name = item[0]
354 name = item[0]
355
355
356 if name=='array dimensions':
356 if name=='array dimensions':
357 table = gp[name][:]
357 table = gp[name][:]
358 listShapes = {}
358 listShapes = {}
359 for shapes in table:
359 for shapes in table:
360 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
360 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
361 else:
361 else:
362 data = gp[name].value
362 data = gp[name].value
363 listMetaname.append(name)
363 listMetaname.append(name)
364 listMetadata.append(data)
364 listMetadata.append(data)
365
365
366 # if name=='type':
366 # if name=='type':
367 # self.__initDataOut(data)
367 # self.__initDataOut(data)
368
368
369 self.listShapes = listShapes
369 self.listShapes = listShapes
370 self.listMetaname = listMetaname
370 self.listMetaname = listMetaname
371 self.listMeta = listMetadata
371 self.listMeta = listMetadata
372
372
373 fp.close()
373 fp.close()
374 return
374 return
375
375
376 def __readData(self):
376 def __readData(self):
377 grp = self.fp['Data']
377 grp = self.fp['Data']
378 listdataname = []
378 listdataname = []
379 listdata = []
379 listdata = []
380
380
381 for item in grp.items():
381 for item in grp.items():
382 name = item[0]
382 name = item[0]
383 listdataname.append(name)
383 listdataname.append(name)
384
384
385 array = self.__setDataArray(grp[name],self.listShapes[name])
385 array = self.__setDataArray(grp[name],self.listShapes[name])
386 listdata.append(array)
386 listdata.append(array)
387
387
388 self.listDataname = listdataname
388 self.listDataname = listdataname
389 self.listData = listdata
389 self.listData = listdata
390 return
390 return
391
391
392 def __setDataArray(self, dataset, shapes):
392 def __setDataArray(self, dataset, shapes):
393
393
394 nDims = shapes[0]
394 nDims = shapes[0]
395
395
396 nDim2 = shapes[1] #Dimension 0
396 nDim2 = shapes[1] #Dimension 0
397
397
398 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
398 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
399
399
400 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
400 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
401
401
402 mode = shapes[4] #Mode of storing
402 mode = shapes[4] #Mode of storing
403
403
404 blockList = self.blockList
404 blockList = self.blockList
405
405
406 blocksPerFile = self.blocksPerFile
406 blocksPerFile = self.blocksPerFile
407
407
408 #Depending on what mode the data was stored
408 #Depending on what mode the data was stored
409 # if mode == 0: #Divided in channels
409 if mode == 0: #Divided in channels
410 # strds = 'channel'
410 arrayData = dataset.value.astype(numpy.float)[0][blockList]
411 # nDatas = nDim2
412 # newShapes = (blocksPerFile,nDim1,nDim0)
413 if mode == 1: #Divided in parameter
411 if mode == 1: #Divided in parameter
414 strds = 'param'
412 strds = 'table'
415 nDatas = nDim1
413 nDatas = nDim1
416 newShapes = (blocksPerFile,nDim2,nDim0)
414 newShapes = (blocksPerFile,nDim2,nDim0)
417 elif mode==2: #Concatenated in a table
415 elif mode==2: #Concatenated in a table
418 strds = 'table0'
416 strds = 'table0'
419 arrayData = dataset[strds].value
417 arrayData = dataset[strds].value
420 #Selecting part of the dataset
418 #Selecting part of the dataset
421 utctime = arrayData[:,0]
419 utctime = arrayData[:,0]
422 u, indices = numpy.unique(utctime, return_index=True)
420 u, indices = numpy.unique(utctime, return_index=True)
423
421
424 if blockList.size != indices.size:
422 if blockList.size != indices.size:
425 indMin = indices[blockList[0]]
423 indMin = indices[blockList[0]]
426 indMax = indices[blockList[-1] + 1]
424 indMax = indices[blockList[-1] + 1]
427 arrayData = arrayData[indMin:indMax,:]
425 arrayData = arrayData[indMin:indMax,:]
428 return arrayData
426 return arrayData
429
427
430 #------- One dimension ---------------
428 #------- One dimension ---------------
431 if nDims == 1:
429 if nDims == 0:
432 arrayData = dataset.value.astype(numpy.float)[0][blockList]
430 arrayData = dataset.value.astype(numpy.float)[0][blockList]
433
431
434 #------- Two dimensions -----------
432 #------- Two dimensions -----------
435 elif nDims == 2:
433 elif nDims == 2:
436 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
434 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
437 newShapes = (blocksPerFile,nDim0)
435 newShapes = (blocksPerFile,nDim0)
438 nDatas = nDim1
436 nDatas = nDim1
439
437
440 for i in range(nDatas):
438 for i in range(nDatas):
441 data = dataset[strds + str(i)].value
439 data = dataset[strds + str(i)].value
442 arrayData[:,i,:] = data[blockList,:]
440 arrayData[:,i,:] = data[blockList,:]
443
441
444 #------- Three dimensions ---------
442 #------- Three dimensions ---------
445 else:
443 else:
446 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
444 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
447 for i in range(nDatas):
445 for i in range(nDatas):
448
446
449 data = dataset[strds + str(i)].value
447 data = dataset[strds + str(i)].value
450 data = data[blockList,:,:]
448 data = data[blockList,:,:]
451 data = data.reshape(newShapes)
449 data = data.reshape(newShapes)
452 # if mode == 0:
450 # if mode == 0:
453 # arrayData[:,i,:,:] = data
451 # arrayData[:,i,:,:] = data
454 # else:
452 # else:
455 arrayData[:,:,i,:] = data
453 arrayData[:,:,i,:] = data
456
454
457 return arrayData
455 return arrayData
458
456
459 def __setDataOut(self):
457 def __setDataOut(self):
460 listMeta = self.listMeta
458 listMeta = self.listMeta
461 listMetaname = self.listMetaname
459 listMetaname = self.listMetaname
462 listDataname = self.listDataname
460 listDataname = self.listDataname
463 listData = self.listData
461 listData = self.listData
464 listShapes = self.listShapes
462 listShapes = self.listShapes
465
463
466 blockIndex = self.blockIndex
464 blockIndex = self.blockIndex
467 # blockList = self.blockList
465 # blockList = self.blockList
468
466
469 for i in range(len(listMeta)):
467 for i in range(len(listMeta)):
470 setattr(self.dataOut,listMetaname[i],listMeta[i])
468 setattr(self.dataOut,listMetaname[i],listMeta[i])
471
469
472 for j in range(len(listData)):
470 for j in range(len(listData)):
473 nShapes = listShapes[listDataname[j]][0]
471 nShapes = listShapes[listDataname[j]][0]
474 mode = listShapes[listDataname[j]][4]
472 mode = listShapes[listDataname[j]][4]
475 if nShapes == 1:
473 if nShapes == 1:
476 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
474 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
477 elif nShapes > 1:
475 elif nShapes > 1:
478 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
476 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
477 elif mode==0:
478 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
479 #Mode Meteors
479 #Mode Meteors
480 elif mode ==2:
480 elif mode ==2:
481 selectedData = self.__selectDataMode2(listData[j], blockIndex)
481 selectedData = self.__selectDataMode2(listData[j], blockIndex)
482 setattr(self.dataOut, listDataname[j], selectedData)
482 setattr(self.dataOut, listDataname[j], selectedData)
483 return
483 return
484
484
485 def __selectDataMode2(self, data, blockIndex):
485 def __selectDataMode2(self, data, blockIndex):
486 utctime = data[:,0]
486 utctime = data[:,0]
487 aux, indices = numpy.unique(utctime, return_inverse=True)
487 aux, indices = numpy.unique(utctime, return_inverse=True)
488 selInd = numpy.where(indices == blockIndex)[0]
488 selInd = numpy.where(indices == blockIndex)[0]
489 selData = data[selInd,:]
489 selData = data[selInd,:]
490
490
491 return selData
491 return selData
492
492
493 def getData(self):
493 def getData(self):
494
494
495 # if self.flagNoMoreFiles:
495 # if self.flagNoMoreFiles:
496 # self.dataOut.flagNoData = True
496 # self.dataOut.flagNoData = True
497 # print 'Process finished'
497 # print 'Process finished'
498 # return 0
498 # return 0
499 #
499 #
500 if self.blockIndex==self.blocksPerFile:
500 if self.blockIndex==self.blocksPerFile:
501 if not( self.__setNextFileOffline() ):
501 if not( self.__setNextFileOffline() ):
502 self.dataOut.flagNoData = True
502 self.dataOut.flagNoData = True
503 return 0
503 return 0
504
504
505 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
505 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
506 # self.dataOut.flagNoData = True
506 # self.dataOut.flagNoData = True
507 # return 0
507 # return 0
508 # self.__readData()
508 # self.__readData()
509 self.__setDataOut()
509 self.__setDataOut()
510 self.dataOut.flagNoData = False
510 self.dataOut.flagNoData = False
511
511
512 self.blockIndex += 1
512 self.blockIndex += 1
513
513
514 return
514 return
515
515
516 def run(self, **kwargs):
516 def run(self, **kwargs):
517
517
518 if not(self.isConfig):
518 if not(self.isConfig):
519 self.setup(**kwargs)
519 self.setup(**kwargs)
520 # self.setObjProperties()
520 # self.setObjProperties()
521 self.isConfig = True
521 self.isConfig = True
522
522
523 self.getData()
523 self.getData()
524
524
525 return
525 return
526
526
527 class HDF5Writer(Operation):
527 class HDF5Writer(Operation):
528 '''
528 '''
529 HDF5 Writer, stores parameters data in HDF5 format files
529 HDF5 Writer, stores parameters data in HDF5 format files
530
530
531 path: path where the files will be stored
531 path: path where the files will be stored
532
532
533 blocksPerFile: number of blocks that will be saved in per HDF5 format file
533 blocksPerFile: number of blocks that will be saved in per HDF5 format file
534
534
535 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
535 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
536
536
537 metadataList: list of attributes that will be stored as metadata
537 metadataList: list of attributes that will be stored as metadata
538
538
539 dataList: list of attributes that will be stores as data
539 dataList: list of attributes that will be stores as data
540
540
541 '''
541 '''
542
542
543
543
544 ext = ".hdf5"
544 ext = ".hdf5"
545
545
546 optchar = "D"
546 optchar = "D"
547
547
548 metaoptchar = "M"
548 metaoptchar = "M"
549
549
550 metaFile = None
550 metaFile = None
551
551
552 filename = None
552 filename = None
553
553
554 path = None
554 path = None
555
555
556 setFile = None
556 setFile = None
557
557
558 fp = None
558 fp = None
559
559
560 grp = None
560 grp = None
561
561
562 ds = None
562 ds = None
563
563
564 firsttime = True
564 firsttime = True
565
565
566 #Configurations
566 #Configurations
567
567
568 blocksPerFile = None
568 blocksPerFile = None
569
569
570 blockIndex = None
570 blockIndex = None
571
571
572 dataOut = None
572 dataOut = None
573
573
574 #Data Arrays
574 #Data Arrays
575
575
576 dataList = None
576 dataList = None
577
577
578 metadataList = None
578 metadataList = None
579
579
580 # arrayDim = None
580 # arrayDim = None
581
581
582 dsList = None #List of dictionaries with dataset properties
582 dsList = None #List of dictionaries with dataset properties
583
583
584 tableDim = None
584 tableDim = None
585
585
586 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
586 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
587
587
588 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
588 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
589
589
590 currentDay = None
590 currentDay = None
591
591
592 def __init__(self):
592 def __init__(self):
593
593
594 Operation.__init__(self)
594 Operation.__init__(self)
595 self.isConfig = False
595 self.isConfig = False
596 return
596 return
597
597
598 def setup(self, dataOut, **kwargs):
598 def setup(self, dataOut, **kwargs):
599
599
600 self.path = kwargs['path']
600 self.path = kwargs['path']
601
601
602 if kwargs.has_key('blocksPerFile'):
602 if kwargs.has_key('blocksPerFile'):
603 self.blocksPerFile = kwargs['blocksPerFile']
603 self.blocksPerFile = kwargs['blocksPerFile']
604 else:
604 else:
605 self.blocksPerFile = 10
605 self.blocksPerFile = 10
606
606
607 self.metadataList = kwargs['metadataList']
607 self.metadataList = kwargs['metadataList']
608 self.dataList = kwargs['dataList']
608 self.dataList = kwargs['dataList']
609 self.dataOut = dataOut
609 self.dataOut = dataOut
610
610
611 if kwargs.has_key('mode'):
611 if kwargs.has_key('mode'):
612 mode = kwargs['mode']
612 mode = kwargs['mode']
613
613
614 if type(mode) == int:
614 if type(mode) == int:
615 mode = numpy.zeros(len(self.dataList)) + mode
615 mode = numpy.zeros(len(self.dataList)) + mode
616 else:
616 else:
617 mode = numpy.ones(len(self.dataList))
617 mode = numpy.ones(len(self.dataList))
618
618
619 self.mode = mode
619 self.mode = mode
620
620
621 arrayDim = numpy.zeros((len(self.dataList),5))
621 arrayDim = numpy.zeros((len(self.dataList),5))
622
622
623 #Table dimensions
623 #Table dimensions
624 dtype0 = self.dtype
624 dtype0 = self.dtype
625 tableList = []
625 tableList = []
626
626
627 #Dictionary and list of tables
627 #Dictionary and list of tables
628 dsList = []
628 dsList = []
629
629
630 for i in range(len(self.dataList)):
630 for i in range(len(self.dataList)):
631 dsDict = {}
631 dsDict = {}
632 dataAux = getattr(self.dataOut, self.dataList[i])
632 dataAux = getattr(self.dataOut, self.dataList[i])
633 dsDict['variable'] = self.dataList[i]
633 dsDict['variable'] = self.dataList[i]
634 #--------------------- Conditionals ------------------------
634 #--------------------- Conditionals ------------------------
635 #There is no data
635 #There is no data
636 if dataAux == None:
636 if dataAux == None:
637 return 0
637 return 0
638
638
639 #Not array, just a number
639 #Not array, just a number
640 #Mode 0
640 #Mode 0
641 if type(dataAux)==float or type(dataAux)==int:
641 if type(dataAux)==float or type(dataAux)==int:
642 dsDict['mode'] = 0
642 dsDict['mode'] = 0
643 dsDict['nDim'] = 0
643 dsDict['nDim'] = 0
644 arrayDim[i,0] = 0
644 arrayDim[i,0] = 0
645 dsList.append(dsDict)
645 dsList.append(dsDict)
646
646
647 #Mode 2: meteors
647 #Mode 2: meteors
648 elif mode[i] == 2:
648 elif mode[i] == 2:
649 # dsDict['nDim'] = 0
649 # dsDict['nDim'] = 0
650 dsDict['dsName'] = 'table0'
650 dsDict['dsName'] = 'table0'
651 dsDict['mode'] = 2 # Mode meteors
651 dsDict['mode'] = 2 # Mode meteors
652 dsDict['shape'] = dataAux.shape[-1]
652 dsDict['shape'] = dataAux.shape[-1]
653 dsDict['nDim'] = 0
653 dsDict['nDim'] = 0
654 dsDict['dsNumber'] = 1
654 dsDict['dsNumber'] = 1
655
655
656 arrayDim[i,3] = dataAux.shape[-1]
656 arrayDim[i,3] = dataAux.shape[-1]
657 arrayDim[i,4] = mode[i] #Mode the data was stored
657 arrayDim[i,4] = mode[i] #Mode the data was stored
658
658
659 dsList.append(dsDict)
659 dsList.append(dsDict)
660
660
661 #Mode 1
661 #Mode 1
662 else:
662 else:
663 arrayDim0 = dataAux.shape #Data dimensions
663 arrayDim0 = dataAux.shape #Data dimensions
664 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
664 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
665 arrayDim[i,4] = mode[i] #Mode the data was stored
665 arrayDim[i,4] = mode[i] #Mode the data was stored
666
666
667 strtable = 'table'
667 strtable = 'table'
668 dsDict['mode'] = 1 # Mode parameters
668 dsDict['mode'] = 1 # Mode parameters
669
669
670 # Three-dimension arrays
670 # Three-dimension arrays
671 if len(arrayDim0) == 3:
671 if len(arrayDim0) == 3:
672 arrayDim[i,1:-1] = numpy.array(arrayDim0)
672 arrayDim[i,1:-1] = numpy.array(arrayDim0)
673 nTables = int(arrayDim[i,2])
673 nTables = int(arrayDim[i,2])
674 dsDict['dsNumber'] = nTables
674 dsDict['dsNumber'] = nTables
675 dsDict['shape'] = arrayDim[i,2:4]
675 dsDict['shape'] = arrayDim[i,2:4]
676 dsDict['nDim'] = 3
676 dsDict['nDim'] = 3
677
677
678 for j in range(nTables):
678 for j in range(nTables):
679 dsDict = dsDict.copy()
679 dsDict = dsDict.copy()
680 dsDict['dsName'] = strtable + str(j)
680 dsDict['dsName'] = strtable + str(j)
681 dsList.append(dsDict)
681 dsList.append(dsDict)
682
682
683 # Two-dimension arrays
683 # Two-dimension arrays
684 elif len(arrayDim0) == 2:
684 elif len(arrayDim0) == 2:
685 arrayDim[i,2:-1] = numpy.array(arrayDim0)
685 arrayDim[i,2:-1] = numpy.array(arrayDim0)
686 nTables = int(arrayDim[i,2])
686 nTables = int(arrayDim[i,2])
687 dsDict['dsNumber'] = nTables
687 dsDict['dsNumber'] = nTables
688 dsDict['shape'] = arrayDim[i,3]
688 dsDict['shape'] = arrayDim[i,3]
689 dsDict['nDim'] = 2
689 dsDict['nDim'] = 2
690
690
691 for j in range(nTables):
691 for j in range(nTables):
692 dsDict = dsDict.copy()
692 dsDict = dsDict.copy()
693 dsDict['dsName'] = strtable + str(j)
693 dsDict['dsName'] = strtable + str(j)
694 dsList.append(dsDict)
694 dsList.append(dsDict)
695
695
696 # One-dimension arrays
696 # One-dimension arrays
697 elif len(arrayDim0) == 1:
697 elif len(arrayDim0) == 1:
698 arrayDim[i,3] = arrayDim0[0]
698 arrayDim[i,3] = arrayDim0[0]
699 dsDict['shape'] = arrayDim0[0]
699 dsDict['shape'] = arrayDim0[0]
700 dsDict['dsNumber'] = 1
700 dsDict['dsNumber'] = 1
701 dsDict['dsName'] = strtable + str(0)
701 dsDict['dsName'] = strtable + str(0)
702 dsDict['nDim'] = 1
702 dsDict['nDim'] = 1
703 dsList.append(dsDict)
703 dsList.append(dsDict)
704
704
705 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
705 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
706 tableList.append(table)
706 tableList.append(table)
707
707
708 # self.arrayDim = arrayDim
708 # self.arrayDim = arrayDim
709 self.dsList = dsList
709 self.dsList = dsList
710 self.tableDim = numpy.array(tableList, dtype = dtype0)
710 self.tableDim = numpy.array(tableList, dtype = dtype0)
711 self.blockIndex = 0
711 self.blockIndex = 0
712
712
713 timeTuple = time.localtime(dataOut.utctime)
713 timeTuple = time.localtime(dataOut.utctime)
714 self.currentDay = timeTuple.tm_yday
714 self.currentDay = timeTuple.tm_yday
715 return 1
715 return 1
716
716
717 def putMetadata(self):
717 def putMetadata(self):
718
718
719 fp = self.createMetadataFile()
719 fp = self.createMetadataFile()
720 self.writeMetadata(fp)
720 self.writeMetadata(fp)
721 fp.close()
721 fp.close()
722 return
722 return
723
723
724 def createMetadataFile(self):
724 def createMetadataFile(self):
725 ext = self.ext
725 ext = self.ext
726 path = self.path
726 path = self.path
727 setFile = self.setFile
727 setFile = self.setFile
728
728
729 timeTuple = time.localtime(self.dataOut.utctime)
729 timeTuple = time.localtime(self.dataOut.utctime)
730
730
731 subfolder = ''
731 subfolder = ''
732 fullpath = os.path.join( path, subfolder )
732 fullpath = os.path.join( path, subfolder )
733
733
734 if not( os.path.exists(fullpath) ):
734 if not( os.path.exists(fullpath) ):
735 os.mkdir(fullpath)
735 os.mkdir(fullpath)
736 setFile = -1 #inicializo mi contador de seteo
736 setFile = -1 #inicializo mi contador de seteo
737
737
738 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
738 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
739 fullpath = os.path.join( path, subfolder )
739 fullpath = os.path.join( path, subfolder )
740
740
741 if not( os.path.exists(fullpath) ):
741 if not( os.path.exists(fullpath) ):
742 os.mkdir(fullpath)
742 os.mkdir(fullpath)
743 setFile = -1 #inicializo mi contador de seteo
743 setFile = -1 #inicializo mi contador de seteo
744
744
745 else:
745 else:
746 filesList = os.listdir( fullpath )
746 filesList = os.listdir( fullpath )
747 filesList = sorted( filesList, key=str.lower )
747 filesList = sorted( filesList, key=str.lower )
748 if len( filesList ) > 0:
748 if len( filesList ) > 0:
749 filesList = [k for k in filesList if 'M' in k]
749 filesList = [k for k in filesList if 'M' in k]
750 filen = filesList[-1]
750 filen = filesList[-1]
751 # el filename debera tener el siguiente formato
751 # el filename debera tener el siguiente formato
752 # 0 1234 567 89A BCDE (hex)
752 # 0 1234 567 89A BCDE (hex)
753 # x YYYY DDD SSS .ext
753 # x YYYY DDD SSS .ext
754 if isNumber( filen[8:11] ):
754 if isNumber( filen[8:11] ):
755 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
755 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
756 else:
756 else:
757 setFile = -1
757 setFile = -1
758 else:
758 else:
759 setFile = -1 #inicializo mi contador de seteo
759 setFile = -1 #inicializo mi contador de seteo
760
760
761 setFile += 1
761 setFile += 1
762
762
763 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
763 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
764 timeTuple.tm_year,
764 timeTuple.tm_year,
765 timeTuple.tm_yday,
765 timeTuple.tm_yday,
766 setFile,
766 setFile,
767 ext )
767 ext )
768
768
769 filename = os.path.join( path, subfolder, file )
769 filename = os.path.join( path, subfolder, file )
770 self.metaFile = file
770 self.metaFile = file
771 #Setting HDF5 File
771 #Setting HDF5 File
772 fp = h5py.File(filename,'w')
772 fp = h5py.File(filename,'w')
773
773
774 return fp
774 return fp
775
775
776 def writeMetadata(self, fp):
776 def writeMetadata(self, fp):
777
777
778 grp = fp.create_group("Metadata")
778 grp = fp.create_group("Metadata")
779 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
779 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
780
780
781 for i in range(len(self.metadataList)):
781 for i in range(len(self.metadataList)):
782 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
782 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
783 return
783 return
784
784
785 def dateFlag(self):
785 def dateFlag(self):
786
786
787 timeTuple = time.localtime(self.dataOut.utctime)
787 timeTuple = time.localtime(self.dataOut.utctime)
788 dataDay = timeTuple.tm_yday
788 dataDay = timeTuple.tm_yday
789
789
790 if dataDay == self.currentDay:
790 if dataDay == self.currentDay:
791 return False
791 return False
792
792
793 self.currentDay = dataDay
793 self.currentDay = dataDay
794 return True
794 return True
795
795
796 def setNextFile(self):
796 def setNextFile(self):
797
797
798 ext = self.ext
798 ext = self.ext
799 path = self.path
799 path = self.path
800 setFile = self.setFile
800 setFile = self.setFile
801 mode = self.mode
801 mode = self.mode
802
802
803 timeTuple = time.localtime(self.dataOut.utctime)
803 timeTuple = time.localtime(self.dataOut.utctime)
804 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
804 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
805
805
806 fullpath = os.path.join( path, subfolder )
806 fullpath = os.path.join( path, subfolder )
807
807
808 if os.path.exists(fullpath):
808 if os.path.exists(fullpath):
809 filesList = os.listdir( fullpath )
809 filesList = os.listdir( fullpath )
810 filesList = [k for k in filesList if 'D' in k]
810 filesList = [k for k in filesList if 'D' in k]
811 if len( filesList ) > 0:
811 if len( filesList ) > 0:
812 filesList = sorted( filesList, key=str.lower )
812 filesList = sorted( filesList, key=str.lower )
813 filen = filesList[-1]
813 filen = filesList[-1]
814 # el filename debera tener el siguiente formato
814 # el filename debera tener el siguiente formato
815 # 0 1234 567 89A BCDE (hex)
815 # 0 1234 567 89A BCDE (hex)
816 # x YYYY DDD SSS .ext
816 # x YYYY DDD SSS .ext
817 if isNumber( filen[8:11] ):
817 if isNumber( filen[8:11] ):
818 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
818 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
819 else:
819 else:
820 setFile = -1
820 setFile = -1
821 else:
821 else:
822 setFile = -1 #inicializo mi contador de seteo
822 setFile = -1 #inicializo mi contador de seteo
823 else:
823 else:
824 os.mkdir(fullpath)
824 os.mkdir(fullpath)
825 setFile = -1 #inicializo mi contador de seteo
825 setFile = -1 #inicializo mi contador de seteo
826
826
827 setFile += 1
827 setFile += 1
828
828
829 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
829 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
830 timeTuple.tm_year,
830 timeTuple.tm_year,
831 timeTuple.tm_yday,
831 timeTuple.tm_yday,
832 setFile,
832 setFile,
833 ext )
833 ext )
834
834
835 filename = os.path.join( path, subfolder, file )
835 filename = os.path.join( path, subfolder, file )
836
836
837 #Setting HDF5 File
837 #Setting HDF5 File
838 fp = h5py.File(filename,'w')
838 fp = h5py.File(filename,'w')
839 #write metadata
839 #write metadata
840 self.writeMetadata(fp)
840 self.writeMetadata(fp)
841 #Write data
841 #Write data
842 grp = fp.create_group("Data")
842 grp = fp.create_group("Data")
843 # grp.attrs['metadata'] = self.metaFile
843 # grp.attrs['metadata'] = self.metaFile
844
844
845 # grp.attrs['blocksPerFile'] = 0
845 # grp.attrs['blocksPerFile'] = 0
846 ds = []
846 ds = []
847 data = []
847 data = []
848 dsList = self.dsList
848 dsList = self.dsList
849 i = 0
849 i = 0
850 while i < len(dsList):
850 while i < len(dsList):
851 dsInfo = dsList[i]
851 dsInfo = dsList[i]
852 #One-dimension data
852 #One-dimension data
853 if dsInfo['mode'] == 0:
853 if dsInfo['mode'] == 0:
854 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
854 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
855 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
855 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
856 ds.append(ds0)
856 ds.append(ds0)
857 data.append([])
857 data.append([])
858 i += 1
858 i += 1
859 continue
859 continue
860 # nDimsForDs.append(nDims[i])
860 # nDimsForDs.append(nDims[i])
861
861
862 elif dsInfo['mode'] == 2:
862 elif dsInfo['mode'] == 2:
863 grp0 = grp.create_group(dsInfo['variable'])
863 grp0 = grp.create_group(dsInfo['variable'])
864 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
864 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
865 ds.append(ds0)
865 ds.append(ds0)
866 data.append([])
866 data.append([])
867 i += 1
867 i += 1
868 continue
868 continue
869
869
870 elif dsInfo['mode'] == 1:
870 elif dsInfo['mode'] == 1:
871 grp0 = grp.create_group(dsInfo['variable'])
871 grp0 = grp.create_group(dsInfo['variable'])
872
872
873 for j in range(dsInfo['dsNumber']):
873 for j in range(dsInfo['dsNumber']):
874 dsInfo = dsList[i]
874 dsInfo = dsList[i]
875 tableName = dsInfo['dsName']
875 tableName = dsInfo['dsName']
876 shape = dsInfo['shape']
876 shape = dsInfo['shape']
877
877
878 if dsInfo['nDim'] == 3:
878 if dsInfo['nDim'] == 3:
879 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
879 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
880 else:
880 else:
881 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
881 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
882
882
883 ds.append(ds0)
883 ds.append(ds0)
884 data.append([])
884 data.append([])
885 i += 1
885 i += 1
886 # nDimsForDs.append(nDims[i])
886 # nDimsForDs.append(nDims[i])
887
887
888 fp.flush()
888 fp.flush()
889 fp.close()
889 fp.close()
890
890
891 # self.nDatas = nDatas
891 # self.nDatas = nDatas
892 # self.nDims = nDims
892 # self.nDims = nDims
893 # self.nDimsForDs = nDimsForDs
893 # self.nDimsForDs = nDimsForDs
894 #Saving variables
894 #Saving variables
895 print 'Writing the file: %s'%filename
895 print 'Writing the file: %s'%filename
896 self.filename = filename
896 self.filename = filename
897 # self.fp = fp
897 # self.fp = fp
898 # self.grp = grp
898 # self.grp = grp
899 # self.grp.attrs.modify('nRecords', 1)
899 # self.grp.attrs.modify('nRecords', 1)
900 self.ds = ds
900 self.ds = ds
901 self.data = data
901 self.data = data
902 # self.setFile = setFile
902 # self.setFile = setFile
903 self.firsttime = True
903 self.firsttime = True
904 self.blockIndex = 0
904 self.blockIndex = 0
905 return
905 return
906
906
907 def putData(self):
907 def putData(self):
908
908
909 if self.blockIndex == self.blocksPerFile or self.dateFlag():
909 if self.blockIndex == self.blocksPerFile or self.dateFlag():
910 self.setNextFile()
910 self.setNextFile()
911
911
912 # if not self.firsttime:
912 # if not self.firsttime:
913 self.readBlock()
913 self.readBlock()
914 self.setBlock() #Prepare data to be written
914 self.setBlock() #Prepare data to be written
915 self.writeBlock() #Write data
915 self.writeBlock() #Write data
916
916
917 return
917 return
918
918
919 def readBlock(self):
919 def readBlock(self):
920
920
921 '''
921 '''
922 data Array configured
922 data Array configured
923
923
924
924
925 self.data
925 self.data
926 '''
926 '''
927 dsList = self.dsList
927 dsList = self.dsList
928 ds = self.ds
928 ds = self.ds
929 #Setting HDF5 File
929 #Setting HDF5 File
930 fp = h5py.File(self.filename,'r+')
930 fp = h5py.File(self.filename,'r+')
931 grp = fp["Data"]
931 grp = fp["Data"]
932 ind = 0
932 ind = 0
933
933
934 # grp.attrs['blocksPerFile'] = 0
934 # grp.attrs['blocksPerFile'] = 0
935 while ind < len(dsList):
935 while ind < len(dsList):
936 dsInfo = dsList[ind]
936 dsInfo = dsList[ind]
937
937
938 if dsInfo['mode'] == 0:
938 if dsInfo['mode'] == 0:
939 ds0 = grp[dsInfo['variable']]
939 ds0 = grp[dsInfo['variable']]
940 ds[ind] = ds0
940 ds[ind] = ds0
941 ind += 1
941 ind += 1
942 else:
942 else:
943
943
944 grp0 = grp[dsInfo['variable']]
944 grp0 = grp[dsInfo['variable']]
945
945
946 for j in range(dsInfo['dsNumber']):
946 for j in range(dsInfo['dsNumber']):
947 dsInfo = dsList[ind]
947 dsInfo = dsList[ind]
948 ds0 = grp0[dsInfo['dsName']]
948 ds0 = grp0[dsInfo['dsName']]
949 ds[ind] = ds0
949 ds[ind] = ds0
950 ind += 1
950 ind += 1
951
951
952 self.fp = fp
952 self.fp = fp
953 self.grp = grp
953 self.grp = grp
954 self.ds = ds
954 self.ds = ds
955
955
956 return
956 return
957
957
958 def setBlock(self):
958 def setBlock(self):
959 '''
959 '''
960 data Array configured
960 data Array configured
961
961
962
962
963 self.data
963 self.data
964 '''
964 '''
965 #Creating Arrays
965 #Creating Arrays
966 dsList = self.dsList
966 dsList = self.dsList
967 data = self.data
967 data = self.data
968 ind = 0
968 ind = 0
969
969
970 while ind < len(dsList):
970 while ind < len(dsList):
971 dsInfo = dsList[ind]
971 dsInfo = dsList[ind]
972 dataAux = getattr(self.dataOut, dsInfo['variable'])
972 dataAux = getattr(self.dataOut, dsInfo['variable'])
973
973
974 mode = dsInfo['mode']
974 mode = dsInfo['mode']
975 nDim = dsInfo['nDim']
975 nDim = dsInfo['nDim']
976
976
977 if mode == 0 or mode == 2 or nDim == 1:
977 if mode == 0 or mode == 2 or nDim == 1:
978 data[ind] = dataAux
978 data[ind] = dataAux
979 ind += 1
979 ind += 1
980 # elif nDim == 1:
980 # elif nDim == 1:
981 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
981 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
982 # ind += 1
982 # ind += 1
983 elif nDim == 2:
983 elif nDim == 2:
984 for j in range(dsInfo['dsNumber']):
984 for j in range(dsInfo['dsNumber']):
985 data[ind] = dataAux[j,:]
985 data[ind] = dataAux[j,:]
986 ind += 1
986 ind += 1
987 elif nDim == 3:
987 elif nDim == 3:
988 for j in range(dsInfo['dsNumber']):
988 for j in range(dsInfo['dsNumber']):
989 data[ind] = dataAux[:,j,:]
989 data[ind] = dataAux[:,j,:]
990 ind += 1
990 ind += 1
991
991
992 self.data = data
992 self.data = data
993 return
993 return
994
994
995 def writeBlock(self):
995 def writeBlock(self):
996 '''
996 '''
997 Saves the block in the HDF5 file
997 Saves the block in the HDF5 file
998 '''
998 '''
999 dsList = self.dsList
999 dsList = self.dsList
1000
1000
1001 for i in range(len(self.ds)):
1001 for i in range(len(self.ds)):
1002 dsInfo = dsList[i]
1002 dsInfo = dsList[i]
1003 nDim = dsInfo['nDim']
1003 nDim = dsInfo['nDim']
1004 mode = dsInfo['mode']
1004 mode = dsInfo['mode']
1005
1005
1006 # First time
1006 # First time
1007 if self.firsttime:
1007 if self.firsttime:
1008 # self.ds[i].resize(self.data[i].shape)
1008 # self.ds[i].resize(self.data[i].shape)
1009 # self.ds[i][self.blockIndex,:] = self.data[i]
1009 # self.ds[i][self.blockIndex,:] = self.data[i]
1010 if type(self.data[i]) == numpy.ndarray:
1010 if type(self.data[i]) == numpy.ndarray:
1011
1011
1012 if nDim == 3:
1012 if nDim == 3:
1013 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1013 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1014 self.ds[i].resize(self.data[i].shape)
1014 self.ds[i].resize(self.data[i].shape)
1015 if mode == 2:
1015 if mode == 2:
1016 self.ds[i].resize(self.data[i].shape)
1016 self.ds[i].resize(self.data[i].shape)
1017 self.ds[i][:] = self.data[i]
1017 self.ds[i][:] = self.data[i]
1018 else:
1018 else:
1019
1019
1020 # From second time
1020 # From second time
1021 # Meteors!
1021 # Meteors!
1022 if mode == 2:
1022 if mode == 2:
1023 dataShape = self.data[i].shape
1023 dataShape = self.data[i].shape
1024 dsShape = self.ds[i].shape
1024 dsShape = self.ds[i].shape
1025 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1025 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1026 self.ds[i][dsShape[0]:,:] = self.data[i]
1026 self.ds[i][dsShape[0]:,:] = self.data[i]
1027 # No dimension
1027 # No dimension
1028 elif mode == 0:
1028 elif mode == 0:
1029 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1029 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1030 self.ds[i][0,-1] = self.data[i]
1030 self.ds[i][0,-1] = self.data[i]
1031 # One dimension
1031 # One dimension
1032 elif nDim == 1:
1032 elif nDim == 1:
1033 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1033 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1034 self.ds[i][-1,:] = self.data[i]
1034 self.ds[i][-1,:] = self.data[i]
1035 # Two dimension
1035 # Two dimension
1036 elif nDim == 2:
1036 elif nDim == 2:
1037 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1037 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1038 self.ds[i][self.blockIndex,:] = self.data[i]
1038 self.ds[i][self.blockIndex,:] = self.data[i]
1039 # Three dimensions
1039 # Three dimensions
1040 elif nDim == 3:
1040 elif nDim == 3:
1041 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1041 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1042 self.ds[i][:,:,-1] = self.data[i]
1042 self.ds[i][:,:,-1] = self.data[i]
1043
1043
1044 self.firsttime = False
1044 self.firsttime = False
1045 self.blockIndex += 1
1045 self.blockIndex += 1
1046
1046
1047 #Close to save changes
1047 #Close to save changes
1048 self.fp.flush()
1048 self.fp.flush()
1049 self.fp.close()
1049 self.fp.close()
1050 return
1050 return
1051
1051
1052 def run(self, dataOut, **kwargs):
1052 def run(self, dataOut, **kwargs):
1053
1053
1054 if not(self.isConfig):
1054 if not(self.isConfig):
1055 flagdata = self.setup(dataOut, **kwargs)
1055 flagdata = self.setup(dataOut, **kwargs)
1056
1056
1057 if not(flagdata):
1057 if not(flagdata):
1058 return
1058 return
1059
1059
1060 self.isConfig = True
1060 self.isConfig = True
1061 # self.putMetadata()
1061 # self.putMetadata()
1062 self.setNextFile()
1062 self.setNextFile()
1063
1063
1064 self.putData()
1064 self.putData()
1065 return
1065 return
1066
1066
1067
1067
General Comments 0
You need to be logged in to leave comments. Login now