##// END OF EJS Templates
schain
RSS
Restore method __setDataArray in ParamReader
Juan C. Valdez -
r872:342bbf5dcbe3
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1029 +1,1092
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class ParamReader(ProcessingUnit):
15 class ParamReader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self):
77 def __init__(self):
78 self.dataOut = Parameters()
78 self.dataOut = Parameters()
79 return
79 return
80
80
81 def setup(self, **kwargs):
81 def setup(self, **kwargs):
82
82
83 path = kwargs['path']
83 path = kwargs['path']
84 startDate = kwargs['startDate']
84 startDate = kwargs['startDate']
85 endDate = kwargs['endDate']
85 endDate = kwargs['endDate']
86 startTime = kwargs['startTime']
86 startTime = kwargs['startTime']
87 endTime = kwargs['endTime']
87 endTime = kwargs['endTime']
88 walk = kwargs['walk']
88 walk = kwargs['walk']
89 if kwargs.has_key('ext'):
89 if kwargs.has_key('ext'):
90 ext = kwargs['ext']
90 ext = kwargs['ext']
91 else:
91 else:
92 ext = '.hdf5'
92 ext = '.hdf5'
93 if kwargs.has_key('timezone'):
93 if kwargs.has_key('timezone'):
94 self.timezone = kwargs['timezone']
94 self.timezone = kwargs['timezone']
95 else:
95 else:
96 self.timezone = 'lt'
96 self.timezone = 'lt'
97
97
98 print "[Reading] Searching files in offline mode ..."
98 print "[Reading] Searching files in offline mode ..."
99 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
99 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 startTime=startTime, endTime=endTime,
100 startTime=startTime, endTime=endTime,
101 ext=ext, walk=walk)
101 ext=ext, walk=walk)
102
102
103 if not(filenameList):
103 if not(filenameList):
104 print "There is no files into the folder: %s"%(path)
104 print "There is no files into the folder: %s"%(path)
105 sys.exit(-1)
105 sys.exit(-1)
106
106
107 self.fileIndex = -1
107 self.fileIndex = -1
108 self.startTime = startTime
108 self.startTime = startTime
109 self.endTime = endTime
109 self.endTime = endTime
110
110
111 self.__readMetadata()
111 self.__readMetadata()
112
112
113 self.__setNextFileOffline()
113 self.__setNextFileOffline()
114
114
115 return
115 return
116
116
117 def __searchFilesOffLine(self,
117 def __searchFilesOffLine(self,
118 path,
118 path,
119 startDate=None,
119 startDate=None,
120 endDate=None,
120 endDate=None,
121 startTime=datetime.time(0,0,0),
121 startTime=datetime.time(0,0,0),
122 endTime=datetime.time(23,59,59),
122 endTime=datetime.time(23,59,59),
123 ext='.hdf5',
123 ext='.hdf5',
124 walk=True):
124 walk=True):
125
125
126 expLabel = ''
126 expLabel = ''
127 self.filenameList = []
127 self.filenameList = []
128 self.datetimeList = []
128 self.datetimeList = []
129
129
130 pathList = []
130 pathList = []
131
131
132 JRODataObj = JRODataReader()
132 JRODataObj = JRODataReader()
133 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
133 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
134
134
135 if dateList == []:
135 if dateList == []:
136 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
136 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
137 datetime.datetime.combine(startDate,startTime).ctime(),
137 datetime.datetime.combine(startDate,startTime).ctime(),
138 datetime.datetime.combine(endDate,endTime).ctime())
138 datetime.datetime.combine(endDate,endTime).ctime())
139
139
140 return None, None
140 return None, None
141
141
142 if len(dateList) > 1:
142 if len(dateList) > 1:
143 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
143 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
144 else:
144 else:
145 print "[Reading] data was found for the date %s" %(dateList[0])
145 print "[Reading] data was found for the date %s" %(dateList[0])
146
146
147 filenameList = []
147 filenameList = []
148 datetimeList = []
148 datetimeList = []
149
149
150 #----------------------------------------------------------------------------------
150 #----------------------------------------------------------------------------------
151
151
152 for thisPath in pathList:
152 for thisPath in pathList:
153 # thisPath = pathList[pathDict[file]]
153 # thisPath = pathList[pathDict[file]]
154
154
155 fileList = glob.glob1(thisPath, "*%s" %ext)
155 fileList = glob.glob1(thisPath, "*%s" %ext)
156 fileList.sort()
156 fileList.sort()
157
157
158 for file in fileList:
158 for file in fileList:
159
159
160 filename = os.path.join(thisPath,file)
160 filename = os.path.join(thisPath,file)
161
161
162 if not isFileInDateRange(filename, startDate, endDate):
162 if not isFileInDateRange(filename, startDate, endDate):
163 continue
163 continue
164
164
165 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
165 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
166
166
167 if not(thisDatetime):
167 if not(thisDatetime):
168 continue
168 continue
169
169
170 filenameList.append(filename)
170 filenameList.append(filename)
171 datetimeList.append(thisDatetime)
171 datetimeList.append(thisDatetime)
172
172
173 if not(filenameList):
173 if not(filenameList):
174 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
174 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
175 return None, None
175 return None, None
176
176
177 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
177 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
178 print
178 print
179
179
180 for i in range(len(filenameList)):
180 for i in range(len(filenameList)):
181 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
181 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
182
182
183 self.filenameList = filenameList
183 self.filenameList = filenameList
184 self.datetimeList = datetimeList
184 self.datetimeList = datetimeList
185
185
186 return pathList, filenameList
186 return pathList, filenameList
187
187
188 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
188 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
189
189
190 """
190 """
191 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
191 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
192
192
193 Inputs:
193 Inputs:
194 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
194 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
195
195
196 startDate : fecha inicial del rango seleccionado en formato datetime.date
196 startDate : fecha inicial del rango seleccionado en formato datetime.date
197
197
198 endDate : fecha final del rango seleccionado en formato datetime.date
198 endDate : fecha final del rango seleccionado en formato datetime.date
199
199
200 startTime : tiempo inicial del rango seleccionado en formato datetime.time
200 startTime : tiempo inicial del rango seleccionado en formato datetime.time
201
201
202 endTime : tiempo final del rango seleccionado en formato datetime.time
202 endTime : tiempo final del rango seleccionado en formato datetime.time
203
203
204 Return:
204 Return:
205 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
205 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
206 fecha especificado, de lo contrario retorna False.
206 fecha especificado, de lo contrario retorna False.
207
207
208 Excepciones:
208 Excepciones:
209 Si el archivo no existe o no puede ser abierto
209 Si el archivo no existe o no puede ser abierto
210 Si la cabecera no puede ser leida.
210 Si la cabecera no puede ser leida.
211
211
212 """
212 """
213
213
214 try:
214 try:
215 fp = h5py.File(filename,'r')
215 fp = h5py.File(filename,'r')
216 grp1 = fp['Data']
216 grp1 = fp['Data']
217
217
218 except IOError:
218 except IOError:
219 traceback.print_exc()
219 traceback.print_exc()
220 raise IOError, "The file %s can't be opened" %(filename)
220 raise IOError, "The file %s can't be opened" %(filename)
221 #chino rata
221 #chino rata
222 #In case has utctime attribute
222 #In case has utctime attribute
223 grp2 = grp1['utctime']
223 grp2 = grp1['utctime']
224 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
224 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
225 thisUtcTime = grp2.value[0]
225 thisUtcTime = grp2.value[0]
226
226
227 fp.close()
227 fp.close()
228
228
229 if self.timezone == 'lt':
229 if self.timezone == 'lt':
230 thisUtcTime -= 5*3600
230 thisUtcTime -= 5*3600
231
231
232 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
232 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
233 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
233 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
234 thisDate = thisDatetime.date()
234 thisDate = thisDatetime.date()
235 thisTime = thisDatetime.time()
235 thisTime = thisDatetime.time()
236
236
237 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
237 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239
239
240 #General case
240 #General case
241 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
241 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
242 #-----------o----------------------------o-----------
242 #-----------o----------------------------o-----------
243 # startTime endTime
243 # startTime endTime
244
244
245 if endTime >= startTime:
245 if endTime >= startTime:
246 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
246 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
247 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
247 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
248 return thisDatetime
248 return thisDatetime
249 return None
249 return None
250
250
251 #If endTime < startTime then endTime belongs to the next day
251 #If endTime < startTime then endTime belongs to the next day
252 #<<<<<<<<<<<o o>>>>>>>>>>>
252 #<<<<<<<<<<<o o>>>>>>>>>>>
253 #-----------o----------------------------o-----------
253 #-----------o----------------------------o-----------
254 # endTime startTime
254 # endTime startTime
255
255
256 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
256 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
257 return None
257 return None
258
258
259 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
259 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
260 return None
260 return None
261
261
262 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
262 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
263 return None
263 return None
264
264
265 return thisDatetime
265 return thisDatetime
266
266
267 def __setNextFileOffline(self):
267 def __setNextFileOffline(self):
268
268
269 self.fileIndex += 1
269 self.fileIndex += 1
270 idFile = self.fileIndex
270 idFile = self.fileIndex
271
271
272 if not(idFile < len(self.filenameList)):
272 if not(idFile < len(self.filenameList)):
273 print "No more Files"
273 print "No more Files"
274 return 0
274 return 0
275
275
276 filename = self.filenameList[idFile]
276 filename = self.filenameList[idFile]
277
277
278 filePointer = h5py.File(filename,'r')
278 filePointer = h5py.File(filename,'r')
279
279
280 self.filename = filename
280 self.filename = filename
281
281
282 self.fp = filePointer
282 self.fp = filePointer
283
283
284 print "Setting the file: %s"%self.filename
284 print "Setting the file: %s"%self.filename
285
285
286 # self.__readMetadata()
286 # self.__readMetadata()
287 self.__setBlockList()
287 self.__setBlockList()
288 self.__readData()
288 self.__readData()
289 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
289 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
290 # self.nRecords = self.fp['Data'].attrs['nRecords']
290 # self.nRecords = self.fp['Data'].attrs['nRecords']
291 self.blockIndex = 0
291 self.blockIndex = 0
292 return 1
292 return 1
293
293
294 def __setBlockList(self):
294 def __setBlockList(self):
295 '''
295 '''
296 Selects the data within the times defined
296 Selects the data within the times defined
297
297
298 self.fp
298 self.fp
299 self.startTime
299 self.startTime
300 self.endTime
300 self.endTime
301
301
302 self.blockList
302 self.blockList
303 self.blocksPerFile
303 self.blocksPerFile
304
304
305 '''
305 '''
306 fp = self.fp
306 fp = self.fp
307 startTime = self.startTime
307 startTime = self.startTime
308 endTime = self.endTime
308 endTime = self.endTime
309
309
310 grp = fp['Data']
310 grp = fp['Data']
311 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
311 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
312
312
313 #ERROOOOR
313 #ERROOOOR
314 if self.timezone == 'lt':
314 if self.timezone == 'lt':
315 thisUtcTime -= 5*3600
315 thisUtcTime -= 5*3600
316
316
317 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
317 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
318
318
319 thisDate = thisDatetime.date()
319 thisDate = thisDatetime.date()
320 thisTime = thisDatetime.time()
320 thisTime = thisDatetime.time()
321
321
322 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
322 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324
324
325 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
325 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
326
326
327 self.blockList = ind
327 self.blockList = ind
328 self.blocksPerFile = len(ind)
328 self.blocksPerFile = len(ind)
329
329
330 return
330 return
331
331
332 def __readMetadata(self):
332 def __readMetadata(self):
333 '''
333 '''
334 Reads Metadata
334 Reads Metadata
335
335
336 self.pathMeta
336 self.pathMeta
337
337
338 self.listShapes
338 self.listShapes
339 self.listMetaname
339 self.listMetaname
340 self.listMeta
340 self.listMeta
341
341
342 '''
342 '''
343
343
344 # grp = self.fp['Data']
344 # grp = self.fp['Data']
345 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
345 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
346 #
346 #
347 # if pathMeta == self.pathMeta:
347 # if pathMeta == self.pathMeta:
348 # return
348 # return
349 # else:
349 # else:
350 # self.pathMeta = pathMeta
350 # self.pathMeta = pathMeta
351 #
351 #
352 # filePointer = h5py.File(self.pathMeta,'r')
352 # filePointer = h5py.File(self.pathMeta,'r')
353 # groupPointer = filePointer['Metadata']
353 # groupPointer = filePointer['Metadata']
354
354
355 filename = self.filenameList[0]
355 filename = self.filenameList[0]
356
356
357 fp = h5py.File(filename,'r')
357 fp = h5py.File(filename,'r')
358
358
359 gp = fp['Metadata']
359 gp = fp['Metadata']
360
360
361 listMetaname = []
361 listMetaname = []
362 listMetadata = []
362 listMetadata = []
363 for item in gp.items():
363 for item in gp.items():
364 name = item[0]
364 name = item[0]
365
365
366 if name=='array dimensions':
366 if name=='array dimensions':
367 table = gp[name][:]
367 table = gp[name][:]
368 listShapes = {}
368 listShapes = {}
369 for shapes in table:
369 for shapes in table:
370 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
370 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
371 else:
371 else:
372 data = gp[name].value
372 data = gp[name].value
373 listMetaname.append(name)
373 listMetaname.append(name)
374 listMetadata.append(data)
374 listMetadata.append(data)
375
375
376 # if name=='type':
376 # if name=='type':
377 # self.__initDataOut(data)
377 # self.__initDataOut(data)
378
378
379 self.listShapes = listShapes
379 self.listShapes = listShapes
380 self.listMetaname = listMetaname
380 self.listMetaname = listMetaname
381 self.listMeta = listMetadata
381 self.listMeta = listMetadata
382
382
383 fp.close()
383 fp.close()
384 return
384 return
385
385
386 def __readData(self):
386 def __readData(self):
387 grp = self.fp['Data']
387 grp = self.fp['Data']
388 listdataname = []
388 listdataname = []
389 listdata = []
389 listdata = []
390
390
391 for item in grp.items():
391 for item in grp.items():
392 name = item[0]
392 name = item[0]
393 listdataname.append(name)
393 listdataname.append(name)
394
394
395 array = self.__setDataArray(grp[name],self.listShapes[name])
395 array = self.__setDataArray(grp[name],self.listShapes[name])
396 listdata.append(array)
396 listdata.append(array)
397
397
398 self.listDataname = listdataname
398 self.listDataname = listdataname
399 self.listData = listdata
399 self.listData = listdata
400 return
400 return
401
401
402 def __setDataArray(self, dataset, shapes):
403
404 nDims = shapes[0]
405
406 nDim2 = shapes[1] #Dimension 0
402
407
408 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409
410 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411
412 mode = shapes[4] #Mode of storing
413
414 blockList = self.blockList
415
416 blocksPerFile = self.blocksPerFile
417
418 #Depending on what mode the data was stored
419 if mode == 0: #Divided in channels
420 arrayData = dataset.value.astype(numpy.float)[0][blockList]
421 if mode == 1: #Divided in parameter
422 strds = 'table'
423 nDatas = nDim1
424 newShapes = (blocksPerFile,nDim2,nDim0)
425 elif mode==2: #Concatenated in a table
426 strds = 'table0'
427 arrayData = dataset[strds].value
428 #Selecting part of the dataset
429 utctime = arrayData[:,0]
430 u, indices = numpy.unique(utctime, return_index=True)
431
432 if blockList.size != indices.size:
433 indMin = indices[blockList[0]]
434 if blockList[1] + 1 >= indices.size:
435 arrayData = arrayData[indMin:,:]
436 else:
437 indMax = indices[blockList[1] + 1]
438 arrayData = arrayData[indMin:indMax,:]
439 return arrayData
440
441 # One dimension
442 if nDims == 0:
443 arrayData = dataset.value.astype(numpy.float)[0][blockList]
444
445 # Two dimensions
446 elif nDims == 2:
447 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448 newShapes = (blocksPerFile,nDim0)
449 nDatas = nDim1
450
451 for i in range(nDatas):
452 data = dataset[strds + str(i)].value
453 arrayData[:,i,:] = data[blockList,:]
454
455 # Three dimensions
456 else:
457 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458 for i in range(nDatas):
459
460 data = dataset[strds + str(i)].value
461
462 for b in range(blockList.size):
463 arrayData[b,:,i,:] = data[:,:,blockList[b]]
464
465 return arrayData
403
466
404 def __setDataOut(self):
467 def __setDataOut(self):
405 listMeta = self.listMeta
468 listMeta = self.listMeta
406 listMetaname = self.listMetaname
469 listMetaname = self.listMetaname
407 listDataname = self.listDataname
470 listDataname = self.listDataname
408 listData = self.listData
471 listData = self.listData
409 listShapes = self.listShapes
472 listShapes = self.listShapes
410
473
411 blockIndex = self.blockIndex
474 blockIndex = self.blockIndex
412 # blockList = self.blockList
475 # blockList = self.blockList
413
476
414 for i in range(len(listMeta)):
477 for i in range(len(listMeta)):
415 setattr(self.dataOut,listMetaname[i],listMeta[i])
478 setattr(self.dataOut,listMetaname[i],listMeta[i])
416
479
417 for j in range(len(listData)):
480 for j in range(len(listData)):
418 nShapes = listShapes[listDataname[j]][0]
481 nShapes = listShapes[listDataname[j]][0]
419 mode = listShapes[listDataname[j]][4]
482 mode = listShapes[listDataname[j]][4]
420 if nShapes == 1:
483 if nShapes == 1:
421 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
484 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
422 elif nShapes > 1:
485 elif nShapes > 1:
423 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
486 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
424 elif mode==0:
487 elif mode==0:
425 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
488 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
426 #Mode Meteors
489 #Mode Meteors
427 elif mode ==2:
490 elif mode ==2:
428 selectedData = self.__selectDataMode2(listData[j], blockIndex)
491 selectedData = self.__selectDataMode2(listData[j], blockIndex)
429 setattr(self.dataOut, listDataname[j], selectedData)
492 setattr(self.dataOut, listDataname[j], selectedData)
430 return
493 return
431
494
432 def __selectDataMode2(self, data, blockIndex):
495 def __selectDataMode2(self, data, blockIndex):
433 utctime = data[:,0]
496 utctime = data[:,0]
434 aux, indices = numpy.unique(utctime, return_inverse=True)
497 aux, indices = numpy.unique(utctime, return_inverse=True)
435 selInd = numpy.where(indices == blockIndex)[0]
498 selInd = numpy.where(indices == blockIndex)[0]
436 selData = data[selInd,:]
499 selData = data[selInd,:]
437
500
438 return selData
501 return selData
439
502
440 def getData(self):
503 def getData(self):
441
504
442 # if self.flagNoMoreFiles:
505 # if self.flagNoMoreFiles:
443 # self.dataOut.flagNoData = True
506 # self.dataOut.flagNoData = True
444 # print 'Process finished'
507 # print 'Process finished'
445 # return 0
508 # return 0
446 #
509 #
447 if self.blockIndex==self.blocksPerFile:
510 if self.blockIndex==self.blocksPerFile:
448 if not( self.__setNextFileOffline() ):
511 if not( self.__setNextFileOffline() ):
449 self.dataOut.flagNoData = True
512 self.dataOut.flagNoData = True
450 return 0
513 return 0
451
514
452 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
515 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
453 # self.dataOut.flagNoData = True
516 # self.dataOut.flagNoData = True
454 # return 0
517 # return 0
455 # self.__readData()
518 # self.__readData()
456 self.__setDataOut()
519 self.__setDataOut()
457 self.dataOut.flagNoData = False
520 self.dataOut.flagNoData = False
458
521
459 self.blockIndex += 1
522 self.blockIndex += 1
460
523
461 return
524 return
462
525
463 def run(self, **kwargs):
526 def run(self, **kwargs):
464
527
465 if not(self.isConfig):
528 if not(self.isConfig):
466 self.setup(**kwargs)
529 self.setup(**kwargs)
467 # self.setObjProperties()
530 # self.setObjProperties()
468 self.isConfig = True
531 self.isConfig = True
469
532
470 self.getData()
533 self.getData()
471
534
472 return
535 return
473
536
474 class ParamWriter(Operation):
537 class ParamWriter(Operation):
475 '''
538 '''
476 HDF5 Writer, stores parameters data in HDF5 format files
539 HDF5 Writer, stores parameters data in HDF5 format files
477
540
478 path: path where the files will be stored
541 path: path where the files will be stored
479
542
480 blocksPerFile: number of blocks that will be saved in per HDF5 format file
543 blocksPerFile: number of blocks that will be saved in per HDF5 format file
481
544
482 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
545 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
483
546
484 metadataList: list of attributes that will be stored as metadata
547 metadataList: list of attributes that will be stored as metadata
485
548
486 dataList: list of attributes that will be stores as data
549 dataList: list of attributes that will be stores as data
487
550
488 '''
551 '''
489
552
490
553
491 ext = ".hdf5"
554 ext = ".hdf5"
492
555
493 optchar = "D"
556 optchar = "D"
494
557
495 metaoptchar = "M"
558 metaoptchar = "M"
496
559
497 metaFile = None
560 metaFile = None
498
561
499 filename = None
562 filename = None
500
563
501 path = None
564 path = None
502
565
503 setFile = None
566 setFile = None
504
567
505 fp = None
568 fp = None
506
569
507 grp = None
570 grp = None
508
571
509 ds = None
572 ds = None
510
573
511 firsttime = True
574 firsttime = True
512
575
513 #Configurations
576 #Configurations
514
577
515 blocksPerFile = None
578 blocksPerFile = None
516
579
517 blockIndex = None
580 blockIndex = None
518
581
519 dataOut = None
582 dataOut = None
520
583
521 #Data Arrays
584 #Data Arrays
522
585
523 dataList = None
586 dataList = None
524
587
525 metadataList = None
588 metadataList = None
526
589
527 # arrayDim = None
590 # arrayDim = None
528
591
529 dsList = None #List of dictionaries with dataset properties
592 dsList = None #List of dictionaries with dataset properties
530
593
531 tableDim = None
594 tableDim = None
532
595
533 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
596 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
534
597
535 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
598 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
536
599
537 currentDay = None
600 currentDay = None
538
601
539 lastTime = None
602 lastTime = None
540
603
541 def __init__(self):
604 def __init__(self):
542
605
543 Operation.__init__(self)
606 Operation.__init__(self)
544 self.isConfig = False
607 self.isConfig = False
545 return
608 return
546
609
547 def setup(self, dataOut, **kwargs):
610 def setup(self, dataOut, **kwargs):
548
611
549 self.path = kwargs['path']
612 self.path = kwargs['path']
550
613
551 if kwargs.has_key('blocksPerFile'):
614 if kwargs.has_key('blocksPerFile'):
552 self.blocksPerFile = kwargs['blocksPerFile']
615 self.blocksPerFile = kwargs['blocksPerFile']
553 else:
616 else:
554 self.blocksPerFile = 10
617 self.blocksPerFile = 10
555
618
556 self.metadataList = kwargs['metadataList']
619 self.metadataList = kwargs['metadataList']
557 self.dataList = kwargs['dataList']
620 self.dataList = kwargs['dataList']
558 self.dataOut = dataOut
621 self.dataOut = dataOut
559
622
560 if kwargs.has_key('mode'):
623 if kwargs.has_key('mode'):
561 mode = kwargs['mode']
624 mode = kwargs['mode']
562
625
563 if type(mode) == int:
626 if type(mode) == int:
564 mode = numpy.zeros(len(self.dataList)) + mode
627 mode = numpy.zeros(len(self.dataList)) + mode
565 else:
628 else:
566 mode = numpy.ones(len(self.dataList))
629 mode = numpy.ones(len(self.dataList))
567
630
568 self.mode = mode
631 self.mode = mode
569
632
570 arrayDim = numpy.zeros((len(self.dataList),5))
633 arrayDim = numpy.zeros((len(self.dataList),5))
571
634
572 #Table dimensions
635 #Table dimensions
573 dtype0 = self.dtype
636 dtype0 = self.dtype
574 tableList = []
637 tableList = []
575
638
576 #Dictionary and list of tables
639 #Dictionary and list of tables
577 dsList = []
640 dsList = []
578
641
579 for i in range(len(self.dataList)):
642 for i in range(len(self.dataList)):
580 dsDict = {}
643 dsDict = {}
581 dataAux = getattr(self.dataOut, self.dataList[i])
644 dataAux = getattr(self.dataOut, self.dataList[i])
582 dsDict['variable'] = self.dataList[i]
645 dsDict['variable'] = self.dataList[i]
583 #--------------------- Conditionals ------------------------
646 #--------------------- Conditionals ------------------------
584 #There is no data
647 #There is no data
585 if dataAux == None:
648 if dataAux == None:
586 return 0
649 return 0
587
650
588 #Not array, just a number
651 #Not array, just a number
589 #Mode 0
652 #Mode 0
590 if type(dataAux)==float or type(dataAux)==int:
653 if type(dataAux)==float or type(dataAux)==int:
591 dsDict['mode'] = 0
654 dsDict['mode'] = 0
592 dsDict['nDim'] = 0
655 dsDict['nDim'] = 0
593 arrayDim[i,0] = 0
656 arrayDim[i,0] = 0
594 dsList.append(dsDict)
657 dsList.append(dsDict)
595
658
596 #Mode 2: meteors
659 #Mode 2: meteors
597 elif mode[i] == 2:
660 elif mode[i] == 2:
598 # dsDict['nDim'] = 0
661 # dsDict['nDim'] = 0
599 dsDict['dsName'] = 'table0'
662 dsDict['dsName'] = 'table0'
600 dsDict['mode'] = 2 # Mode meteors
663 dsDict['mode'] = 2 # Mode meteors
601 dsDict['shape'] = dataAux.shape[-1]
664 dsDict['shape'] = dataAux.shape[-1]
602 dsDict['nDim'] = 0
665 dsDict['nDim'] = 0
603 dsDict['dsNumber'] = 1
666 dsDict['dsNumber'] = 1
604
667
605 arrayDim[i,3] = dataAux.shape[-1]
668 arrayDim[i,3] = dataAux.shape[-1]
606 arrayDim[i,4] = mode[i] #Mode the data was stored
669 arrayDim[i,4] = mode[i] #Mode the data was stored
607
670
608 dsList.append(dsDict)
671 dsList.append(dsDict)
609
672
610 #Mode 1
673 #Mode 1
611 else:
674 else:
612 arrayDim0 = dataAux.shape #Data dimensions
675 arrayDim0 = dataAux.shape #Data dimensions
613 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
676 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
614 arrayDim[i,4] = mode[i] #Mode the data was stored
677 arrayDim[i,4] = mode[i] #Mode the data was stored
615
678
616 strtable = 'table'
679 strtable = 'table'
617 dsDict['mode'] = 1 # Mode parameters
680 dsDict['mode'] = 1 # Mode parameters
618
681
619 # Three-dimension arrays
682 # Three-dimension arrays
620 if len(arrayDim0) == 3:
683 if len(arrayDim0) == 3:
621 arrayDim[i,1:-1] = numpy.array(arrayDim0)
684 arrayDim[i,1:-1] = numpy.array(arrayDim0)
622 nTables = int(arrayDim[i,2])
685 nTables = int(arrayDim[i,2])
623 dsDict['dsNumber'] = nTables
686 dsDict['dsNumber'] = nTables
624 dsDict['shape'] = arrayDim[i,2:4]
687 dsDict['shape'] = arrayDim[i,2:4]
625 dsDict['nDim'] = 3
688 dsDict['nDim'] = 3
626
689
627 for j in range(nTables):
690 for j in range(nTables):
628 dsDict = dsDict.copy()
691 dsDict = dsDict.copy()
629 dsDict['dsName'] = strtable + str(j)
692 dsDict['dsName'] = strtable + str(j)
630 dsList.append(dsDict)
693 dsList.append(dsDict)
631
694
632 # Two-dimension arrays
695 # Two-dimension arrays
633 elif len(arrayDim0) == 2:
696 elif len(arrayDim0) == 2:
634 arrayDim[i,2:-1] = numpy.array(arrayDim0)
697 arrayDim[i,2:-1] = numpy.array(arrayDim0)
635 nTables = int(arrayDim[i,2])
698 nTables = int(arrayDim[i,2])
636 dsDict['dsNumber'] = nTables
699 dsDict['dsNumber'] = nTables
637 dsDict['shape'] = arrayDim[i,3]
700 dsDict['shape'] = arrayDim[i,3]
638 dsDict['nDim'] = 2
701 dsDict['nDim'] = 2
639
702
640 for j in range(nTables):
703 for j in range(nTables):
641 dsDict = dsDict.copy()
704 dsDict = dsDict.copy()
642 dsDict['dsName'] = strtable + str(j)
705 dsDict['dsName'] = strtable + str(j)
643 dsList.append(dsDict)
706 dsList.append(dsDict)
644
707
645 # One-dimension arrays
708 # One-dimension arrays
646 elif len(arrayDim0) == 1:
709 elif len(arrayDim0) == 1:
647 arrayDim[i,3] = arrayDim0[0]
710 arrayDim[i,3] = arrayDim0[0]
648 dsDict['shape'] = arrayDim0[0]
711 dsDict['shape'] = arrayDim0[0]
649 dsDict['dsNumber'] = 1
712 dsDict['dsNumber'] = 1
650 dsDict['dsName'] = strtable + str(0)
713 dsDict['dsName'] = strtable + str(0)
651 dsDict['nDim'] = 1
714 dsDict['nDim'] = 1
652 dsList.append(dsDict)
715 dsList.append(dsDict)
653
716
654 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
717 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
655 tableList.append(table)
718 tableList.append(table)
656
719
657 # self.arrayDim = arrayDim
720 # self.arrayDim = arrayDim
658 self.dsList = dsList
721 self.dsList = dsList
659 self.tableDim = numpy.array(tableList, dtype = dtype0)
722 self.tableDim = numpy.array(tableList, dtype = dtype0)
660 self.blockIndex = 0
723 self.blockIndex = 0
661
724
662 timeTuple = time.localtime(dataOut.utctime)
725 timeTuple = time.localtime(dataOut.utctime)
663 self.currentDay = timeTuple.tm_yday
726 self.currentDay = timeTuple.tm_yday
664 return 1
727 return 1
665
728
666 def putMetadata(self):
729 def putMetadata(self):
667
730
668 fp = self.createMetadataFile()
731 fp = self.createMetadataFile()
669 self.writeMetadata(fp)
732 self.writeMetadata(fp)
670 fp.close()
733 fp.close()
671 return
734 return
672
735
673 def createMetadataFile(self):
736 def createMetadataFile(self):
674 ext = self.ext
737 ext = self.ext
675 path = self.path
738 path = self.path
676 setFile = self.setFile
739 setFile = self.setFile
677
740
678 timeTuple = time.localtime(self.dataOut.utctime)
741 timeTuple = time.localtime(self.dataOut.utctime)
679
742
680 subfolder = ''
743 subfolder = ''
681 fullpath = os.path.join( path, subfolder )
744 fullpath = os.path.join( path, subfolder )
682
745
683 if not( os.path.exists(fullpath) ):
746 if not( os.path.exists(fullpath) ):
684 os.mkdir(fullpath)
747 os.mkdir(fullpath)
685 setFile = -1 #inicializo mi contador de seteo
748 setFile = -1 #inicializo mi contador de seteo
686
749
687 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
750 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
688 fullpath = os.path.join( path, subfolder )
751 fullpath = os.path.join( path, subfolder )
689
752
690 if not( os.path.exists(fullpath) ):
753 if not( os.path.exists(fullpath) ):
691 os.mkdir(fullpath)
754 os.mkdir(fullpath)
692 setFile = -1 #inicializo mi contador de seteo
755 setFile = -1 #inicializo mi contador de seteo
693
756
694 else:
757 else:
695 filesList = os.listdir( fullpath )
758 filesList = os.listdir( fullpath )
696 filesList = sorted( filesList, key=str.lower )
759 filesList = sorted( filesList, key=str.lower )
697 if len( filesList ) > 0:
760 if len( filesList ) > 0:
698 filesList = [k for k in filesList if 'M' in k]
761 filesList = [k for k in filesList if 'M' in k]
699 filen = filesList[-1]
762 filen = filesList[-1]
700 # el filename debera tener el siguiente formato
763 # el filename debera tener el siguiente formato
701 # 0 1234 567 89A BCDE (hex)
764 # 0 1234 567 89A BCDE (hex)
702 # x YYYY DDD SSS .ext
765 # x YYYY DDD SSS .ext
703 if isNumber( filen[8:11] ):
766 if isNumber( filen[8:11] ):
704 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
767 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
705 else:
768 else:
706 setFile = -1
769 setFile = -1
707 else:
770 else:
708 setFile = -1 #inicializo mi contador de seteo
771 setFile = -1 #inicializo mi contador de seteo
709
772
710 setFile += 1
773 setFile += 1
711
774
712 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
775 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
713 timeTuple.tm_year,
776 timeTuple.tm_year,
714 timeTuple.tm_yday,
777 timeTuple.tm_yday,
715 setFile,
778 setFile,
716 ext )
779 ext )
717
780
718 filename = os.path.join( path, subfolder, file )
781 filename = os.path.join( path, subfolder, file )
719 self.metaFile = file
782 self.metaFile = file
720 #Setting HDF5 File
783 #Setting HDF5 File
721 fp = h5py.File(filename,'w')
784 fp = h5py.File(filename,'w')
722
785
723 return fp
786 return fp
724
787
725 def writeMetadata(self, fp):
788 def writeMetadata(self, fp):
726
789
727 grp = fp.create_group("Metadata")
790 grp = fp.create_group("Metadata")
728 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
791 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
729
792
730 for i in range(len(self.metadataList)):
793 for i in range(len(self.metadataList)):
731 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
794 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
732 return
795 return
733
796
734 def timeFlag(self):
797 def timeFlag(self):
735 currentTime = self.dataOut.utctime
798 currentTime = self.dataOut.utctime
736
799
737 if self.lastTime is None:
800 if self.lastTime is None:
738 self.lastTime = currentTime
801 self.lastTime = currentTime
739
802
740 #Day
803 #Day
741 timeTuple = time.localtime(currentTime)
804 timeTuple = time.localtime(currentTime)
742 dataDay = timeTuple.tm_yday
805 dataDay = timeTuple.tm_yday
743
806
744 #Time
807 #Time
745 timeDiff = currentTime - self.lastTime
808 timeDiff = currentTime - self.lastTime
746
809
747 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
810 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
748 if dataDay != self.currentDay:
811 if dataDay != self.currentDay:
749 self.currentDay = dataDay
812 self.currentDay = dataDay
750 return True
813 return True
751 elif timeDiff > 3*60*60:
814 elif timeDiff > 3*60*60:
752 self.lastTime = currentTime
815 self.lastTime = currentTime
753 return True
816 return True
754 else:
817 else:
755 self.lastTime = currentTime
818 self.lastTime = currentTime
756 return False
819 return False
757
820
758 def setNextFile(self):
821 def setNextFile(self):
759
822
760 ext = self.ext
823 ext = self.ext
761 path = self.path
824 path = self.path
762 setFile = self.setFile
825 setFile = self.setFile
763 mode = self.mode
826 mode = self.mode
764
827
765 timeTuple = time.localtime(self.dataOut.utctime)
828 timeTuple = time.localtime(self.dataOut.utctime)
766 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
829 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
767
830
768 fullpath = os.path.join( path, subfolder )
831 fullpath = os.path.join( path, subfolder )
769
832
770 if os.path.exists(fullpath):
833 if os.path.exists(fullpath):
771 filesList = os.listdir( fullpath )
834 filesList = os.listdir( fullpath )
772 filesList = [k for k in filesList if 'D' in k]
835 filesList = [k for k in filesList if 'D' in k]
773 if len( filesList ) > 0:
836 if len( filesList ) > 0:
774 filesList = sorted( filesList, key=str.lower )
837 filesList = sorted( filesList, key=str.lower )
775 filen = filesList[-1]
838 filen = filesList[-1]
776 # el filename debera tener el siguiente formato
839 # el filename debera tener el siguiente formato
777 # 0 1234 567 89A BCDE (hex)
840 # 0 1234 567 89A BCDE (hex)
778 # x YYYY DDD SSS .ext
841 # x YYYY DDD SSS .ext
779 if isNumber( filen[8:11] ):
842 if isNumber( filen[8:11] ):
780 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
843 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
781 else:
844 else:
782 setFile = -1
845 setFile = -1
783 else:
846 else:
784 setFile = -1 #inicializo mi contador de seteo
847 setFile = -1 #inicializo mi contador de seteo
785 else:
848 else:
786 os.mkdir(fullpath)
849 os.mkdir(fullpath)
787 setFile = -1 #inicializo mi contador de seteo
850 setFile = -1 #inicializo mi contador de seteo
788
851
789 setFile += 1
852 setFile += 1
790
853
791 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
854 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
792 timeTuple.tm_year,
855 timeTuple.tm_year,
793 timeTuple.tm_yday,
856 timeTuple.tm_yday,
794 setFile,
857 setFile,
795 ext )
858 ext )
796
859
797 filename = os.path.join( path, subfolder, file )
860 filename = os.path.join( path, subfolder, file )
798
861
799 #Setting HDF5 File
862 #Setting HDF5 File
800 fp = h5py.File(filename,'w')
863 fp = h5py.File(filename,'w')
801 #write metadata
864 #write metadata
802 self.writeMetadata(fp)
865 self.writeMetadata(fp)
803 #Write data
866 #Write data
804 grp = fp.create_group("Data")
867 grp = fp.create_group("Data")
805 # grp.attrs['metadata'] = self.metaFile
868 # grp.attrs['metadata'] = self.metaFile
806
869
807 # grp.attrs['blocksPerFile'] = 0
870 # grp.attrs['blocksPerFile'] = 0
808 ds = []
871 ds = []
809 data = []
872 data = []
810 dsList = self.dsList
873 dsList = self.dsList
811 i = 0
874 i = 0
812 while i < len(dsList):
875 while i < len(dsList):
813 dsInfo = dsList[i]
876 dsInfo = dsList[i]
814 #One-dimension data
877 #One-dimension data
815 if dsInfo['mode'] == 0:
878 if dsInfo['mode'] == 0:
816 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
879 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
817 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
880 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
818 ds.append(ds0)
881 ds.append(ds0)
819 data.append([])
882 data.append([])
820 i += 1
883 i += 1
821 continue
884 continue
822 # nDimsForDs.append(nDims[i])
885 # nDimsForDs.append(nDims[i])
823
886
824 elif dsInfo['mode'] == 2:
887 elif dsInfo['mode'] == 2:
825 grp0 = grp.create_group(dsInfo['variable'])
888 grp0 = grp.create_group(dsInfo['variable'])
826 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
889 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
827 ds.append(ds0)
890 ds.append(ds0)
828 data.append([])
891 data.append([])
829 i += 1
892 i += 1
830 continue
893 continue
831
894
832 elif dsInfo['mode'] == 1:
895 elif dsInfo['mode'] == 1:
833 grp0 = grp.create_group(dsInfo['variable'])
896 grp0 = grp.create_group(dsInfo['variable'])
834
897
835 for j in range(dsInfo['dsNumber']):
898 for j in range(dsInfo['dsNumber']):
836 dsInfo = dsList[i]
899 dsInfo = dsList[i]
837 tableName = dsInfo['dsName']
900 tableName = dsInfo['dsName']
838 shape = dsInfo['shape']
901 shape = dsInfo['shape']
839
902
840 if dsInfo['nDim'] == 3:
903 if dsInfo['nDim'] == 3:
841 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
904 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
842 else:
905 else:
843 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
906 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
844
907
845 ds.append(ds0)
908 ds.append(ds0)
846 data.append([])
909 data.append([])
847 i += 1
910 i += 1
848 # nDimsForDs.append(nDims[i])
911 # nDimsForDs.append(nDims[i])
849
912
850 fp.flush()
913 fp.flush()
851 fp.close()
914 fp.close()
852
915
853 # self.nDatas = nDatas
916 # self.nDatas = nDatas
854 # self.nDims = nDims
917 # self.nDims = nDims
855 # self.nDimsForDs = nDimsForDs
918 # self.nDimsForDs = nDimsForDs
856 #Saving variables
919 #Saving variables
857 print 'Writing the file: %s'%filename
920 print 'Writing the file: %s'%filename
858 self.filename = filename
921 self.filename = filename
859 # self.fp = fp
922 # self.fp = fp
860 # self.grp = grp
923 # self.grp = grp
861 # self.grp.attrs.modify('nRecords', 1)
924 # self.grp.attrs.modify('nRecords', 1)
862 self.ds = ds
925 self.ds = ds
863 self.data = data
926 self.data = data
864 # self.setFile = setFile
927 # self.setFile = setFile
865 self.firsttime = True
928 self.firsttime = True
866 self.blockIndex = 0
929 self.blockIndex = 0
867 return
930 return
868
931
869 def putData(self):
932 def putData(self):
870
933
871 if self.blockIndex == self.blocksPerFile or self.timeFlag():
934 if self.blockIndex == self.blocksPerFile or self.timeFlag():
872 self.setNextFile()
935 self.setNextFile()
873
936
874 # if not self.firsttime:
937 # if not self.firsttime:
875 self.readBlock()
938 self.readBlock()
876 self.setBlock() #Prepare data to be written
939 self.setBlock() #Prepare data to be written
877 self.writeBlock() #Write data
940 self.writeBlock() #Write data
878
941
879 return
942 return
880
943
881 def readBlock(self):
944 def readBlock(self):
882
945
883 '''
946 '''
884 data Array configured
947 data Array configured
885
948
886
949
887 self.data
950 self.data
888 '''
951 '''
889 dsList = self.dsList
952 dsList = self.dsList
890 ds = self.ds
953 ds = self.ds
891 #Setting HDF5 File
954 #Setting HDF5 File
892 fp = h5py.File(self.filename,'r+')
955 fp = h5py.File(self.filename,'r+')
893 grp = fp["Data"]
956 grp = fp["Data"]
894 ind = 0
957 ind = 0
895
958
896 # grp.attrs['blocksPerFile'] = 0
959 # grp.attrs['blocksPerFile'] = 0
897 while ind < len(dsList):
960 while ind < len(dsList):
898 dsInfo = dsList[ind]
961 dsInfo = dsList[ind]
899
962
900 if dsInfo['mode'] == 0:
963 if dsInfo['mode'] == 0:
901 ds0 = grp[dsInfo['variable']]
964 ds0 = grp[dsInfo['variable']]
902 ds[ind] = ds0
965 ds[ind] = ds0
903 ind += 1
966 ind += 1
904 else:
967 else:
905
968
906 grp0 = grp[dsInfo['variable']]
969 grp0 = grp[dsInfo['variable']]
907
970
908 for j in range(dsInfo['dsNumber']):
971 for j in range(dsInfo['dsNumber']):
909 dsInfo = dsList[ind]
972 dsInfo = dsList[ind]
910 ds0 = grp0[dsInfo['dsName']]
973 ds0 = grp0[dsInfo['dsName']]
911 ds[ind] = ds0
974 ds[ind] = ds0
912 ind += 1
975 ind += 1
913
976
914 self.fp = fp
977 self.fp = fp
915 self.grp = grp
978 self.grp = grp
916 self.ds = ds
979 self.ds = ds
917
980
918 return
981 return
919
982
920 def setBlock(self):
983 def setBlock(self):
921 '''
984 '''
922 data Array configured
985 data Array configured
923
986
924
987
925 self.data
988 self.data
926 '''
989 '''
927 #Creating Arrays
990 #Creating Arrays
928 dsList = self.dsList
991 dsList = self.dsList
929 data = self.data
992 data = self.data
930 ind = 0
993 ind = 0
931
994
932 while ind < len(dsList):
995 while ind < len(dsList):
933 dsInfo = dsList[ind]
996 dsInfo = dsList[ind]
934 dataAux = getattr(self.dataOut, dsInfo['variable'])
997 dataAux = getattr(self.dataOut, dsInfo['variable'])
935
998
936 mode = dsInfo['mode']
999 mode = dsInfo['mode']
937 nDim = dsInfo['nDim']
1000 nDim = dsInfo['nDim']
938
1001
939 if mode == 0 or mode == 2 or nDim == 1:
1002 if mode == 0 or mode == 2 or nDim == 1:
940 data[ind] = dataAux
1003 data[ind] = dataAux
941 ind += 1
1004 ind += 1
942 # elif nDim == 1:
1005 # elif nDim == 1:
943 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1006 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
944 # ind += 1
1007 # ind += 1
945 elif nDim == 2:
1008 elif nDim == 2:
946 for j in range(dsInfo['dsNumber']):
1009 for j in range(dsInfo['dsNumber']):
947 data[ind] = dataAux[j,:]
1010 data[ind] = dataAux[j,:]
948 ind += 1
1011 ind += 1
949 elif nDim == 3:
1012 elif nDim == 3:
950 for j in range(dsInfo['dsNumber']):
1013 for j in range(dsInfo['dsNumber']):
951 data[ind] = dataAux[:,j,:]
1014 data[ind] = dataAux[:,j,:]
952 ind += 1
1015 ind += 1
953
1016
954 self.data = data
1017 self.data = data
955 return
1018 return
956
1019
957 def writeBlock(self):
1020 def writeBlock(self):
958 '''
1021 '''
959 Saves the block in the HDF5 file
1022 Saves the block in the HDF5 file
960 '''
1023 '''
961 dsList = self.dsList
1024 dsList = self.dsList
962
1025
963 for i in range(len(self.ds)):
1026 for i in range(len(self.ds)):
964 dsInfo = dsList[i]
1027 dsInfo = dsList[i]
965 nDim = dsInfo['nDim']
1028 nDim = dsInfo['nDim']
966 mode = dsInfo['mode']
1029 mode = dsInfo['mode']
967
1030
968 # First time
1031 # First time
969 if self.firsttime:
1032 if self.firsttime:
970 # self.ds[i].resize(self.data[i].shape)
1033 # self.ds[i].resize(self.data[i].shape)
971 # self.ds[i][self.blockIndex,:] = self.data[i]
1034 # self.ds[i][self.blockIndex,:] = self.data[i]
972 if type(self.data[i]) == numpy.ndarray:
1035 if type(self.data[i]) == numpy.ndarray:
973
1036
974 if nDim == 3:
1037 if nDim == 3:
975 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1038 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
976 self.ds[i].resize(self.data[i].shape)
1039 self.ds[i].resize(self.data[i].shape)
977 if mode == 2:
1040 if mode == 2:
978 self.ds[i].resize(self.data[i].shape)
1041 self.ds[i].resize(self.data[i].shape)
979 self.ds[i][:] = self.data[i]
1042 self.ds[i][:] = self.data[i]
980 else:
1043 else:
981
1044
982 # From second time
1045 # From second time
983 # Meteors!
1046 # Meteors!
984 if mode == 2:
1047 if mode == 2:
985 dataShape = self.data[i].shape
1048 dataShape = self.data[i].shape
986 dsShape = self.ds[i].shape
1049 dsShape = self.ds[i].shape
987 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1050 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
988 self.ds[i][dsShape[0]:,:] = self.data[i]
1051 self.ds[i][dsShape[0]:,:] = self.data[i]
989 # No dimension
1052 # No dimension
990 elif mode == 0:
1053 elif mode == 0:
991 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1054 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
992 self.ds[i][0,-1] = self.data[i]
1055 self.ds[i][0,-1] = self.data[i]
993 # One dimension
1056 # One dimension
994 elif nDim == 1:
1057 elif nDim == 1:
995 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1058 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
996 self.ds[i][-1,:] = self.data[i]
1059 self.ds[i][-1,:] = self.data[i]
997 # Two dimension
1060 # Two dimension
998 elif nDim == 2:
1061 elif nDim == 2:
999 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1062 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1000 self.ds[i][self.blockIndex,:] = self.data[i]
1063 self.ds[i][self.blockIndex,:] = self.data[i]
1001 # Three dimensions
1064 # Three dimensions
1002 elif nDim == 3:
1065 elif nDim == 3:
1003 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1066 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1004 self.ds[i][:,:,-1] = self.data[i]
1067 self.ds[i][:,:,-1] = self.data[i]
1005
1068
1006 self.firsttime = False
1069 self.firsttime = False
1007 self.blockIndex += 1
1070 self.blockIndex += 1
1008
1071
1009 #Close to save changes
1072 #Close to save changes
1010 self.fp.flush()
1073 self.fp.flush()
1011 self.fp.close()
1074 self.fp.close()
1012 return
1075 return
1013
1076
1014 def run(self, dataOut, **kwargs):
1077 def run(self, dataOut, **kwargs):
1015
1078
1016 if not(self.isConfig):
1079 if not(self.isConfig):
1017 flagdata = self.setup(dataOut, **kwargs)
1080 flagdata = self.setup(dataOut, **kwargs)
1018
1081
1019 if not(flagdata):
1082 if not(flagdata):
1020 return
1083 return
1021
1084
1022 self.isConfig = True
1085 self.isConfig = True
1023 # self.putMetadata()
1086 # self.putMetadata()
1024 self.setNextFile()
1087 self.setNextFile()
1025
1088
1026 self.putData()
1089 self.putData()
1027 return
1090 return
1028
1091
1029
1092
General Comments 0
You need to be logged in to leave comments. Login now