##// END OF EJS Templates
schain
RSS
-Bug fixed in HDF5 writing module...
Julio Valdez -
r724:fb11662812fb
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1004 +1,1026
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6
6
7 from schainpy.model.data.jrodata import *
7 from schainpy.model.data.jrodata import *
8 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
8 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.io.jroIO_base import *
9 from schainpy.model.io.jroIO_base import *
10
10
11
11
12 class HDF5Reader(ProcessingUnit):
12 class HDF5Reader(ProcessingUnit):
13
13
14 ext = ".hdf5"
14 ext = ".hdf5"
15
15
16 optchar = "D"
16 optchar = "D"
17
17
18 timezone = None
18 timezone = None
19
19
20 secStart = None
20 secStart = None
21
21
22 secEnd = None
22 secEnd = None
23
23
24 fileIndex = None
24 fileIndex = None
25
25
26 blockIndex = None
26 blockIndex = None
27
27
28 blocksPerFile = None
28 blocksPerFile = None
29
29
30 path = None
30 path = None
31
31
32 #List of Files
32 #List of Files
33
33
34 filenameList = None
34 filenameList = None
35
35
36 datetimeList = None
36 datetimeList = None
37
37
38 #Hdf5 File
38 #Hdf5 File
39
39
40 fpMetadata = None
40 fpMetadata = None
41
41
42 pathMeta = None
42 pathMeta = None
43
43
44 listMetaname = None
44 listMetaname = None
45
45
46 listMeta = None
46 listMeta = None
47
47
48 listDataname = None
48 listDataname = None
49
49
50 listData = None
50 listData = None
51
51
52 listShapes = None
52 listShapes = None
53
53
54 fp = None
54 fp = None
55
55
56 #dataOut reconstruction
56 #dataOut reconstruction
57
57
58 dataOut = None
58 dataOut = None
59
59
60 nRecords = None
60 nRecords = None
61
61
62
62
63 def __init__(self):
63 def __init__(self):
64 self.dataOut = self.__createObjByDefault()
64 self.dataOut = self.__createObjByDefault()
65 return
65 return
66
66
67 def __createObjByDefault(self):
67 def __createObjByDefault(self):
68
68
69 dataObj = Parameters()
69 dataObj = Parameters()
70
70
71 return dataObj
71 return dataObj
72
72
73 def setup(self,path=None,
73 def setup(self,path=None,
74 startDate=None,
74 startDate=None,
75 endDate=None,
75 endDate=None,
76 startTime=datetime.time(0,0,0),
76 startTime=datetime.time(0,0,0),
77 endTime=datetime.time(23,59,59),
77 endTime=datetime.time(23,59,59),
78 walk=True,
78 walk=True,
79 timezone='ut',
79 timezone='ut',
80 all=0,
80 all=0,
81 online=False,
81 online=False,
82 ext=None):
82 ext=None):
83
83
84 if ext==None:
84 if ext==None:
85 ext = self.ext
85 ext = self.ext
86 self.timezone = timezone
86 self.timezone = timezone
87 # self.all = all
87 # self.all = all
88 # self.online = online
88 # self.online = online
89 self.path = path
89 self.path = path
90
90
91 startDateTime = datetime.datetime.combine(startDate,startTime)
91 startDateTime = datetime.datetime.combine(startDate,startTime)
92 endDateTime = datetime.datetime.combine(endDate,endTime)
92 endDateTime = datetime.datetime.combine(endDate,endTime)
93 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
93 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
94 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
94 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
95
95
96 self.secStart = secStart
96 self.secStart = secStart
97 self.secEnd = secEnd
97 self.secEnd = secEnd
98
98
99 if not(online):
99 if not(online):
100 #Busqueda de archivos offline
100 #Busqueda de archivos offline
101 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
101 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
102 else:
102 else:
103 self.__searchFilesOnline(path, walk)
103 self.__searchFilesOnline(path, walk)
104
104
105 if not(self.filenameList):
105 if not(self.filenameList):
106 print "There is no files into the folder: %s"%(path)
106 print "There is no files into the folder: %s"%(path)
107 sys.exit(-1)
107 sys.exit(-1)
108
108
109 # self.__getExpParameters()
109 # self.__getExpParameters()
110
110
111 self.fileIndex = -1
111 self.fileIndex = -1
112
112
113 self.__setNextFileOffline()
113 self.__setNextFileOffline()
114
114
115 self.__readMetadata()
115 self.__readMetadata()
116
116
117 self.blockIndex = 0
117 self.blockIndex = 0
118
118
119 return
119 return
120
120
121 def __searchFilesOffline(self,
121 def __searchFilesOffline(self,
122 path,
122 path,
123 startDate,
123 startDate,
124 endDate,
124 endDate,
125 ext,
125 ext,
126 startTime=datetime.time(0,0,0),
126 startTime=datetime.time(0,0,0),
127 endTime=datetime.time(23,59,59),
127 endTime=datetime.time(23,59,59),
128 secStart = 0,
128 secStart = 0,
129 secEnd = numpy.inf,
129 secEnd = numpy.inf,
130 walk=True):
130 walk=True):
131
131
132 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
132 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
133 #
133 #
134 # self.__checkPath()
134 # self.__checkPath()
135 #
135 #
136 # self.__findDataForDates()
136 # self.__findDataForDates()
137 #
137 #
138 # self.__selectDataForTimes()
138 # self.__selectDataForTimes()
139 #
139 #
140 # for i in range(len(self.filenameList)):
140 # for i in range(len(self.filenameList)):
141 # print "%s" %(self.filenameList[i])
141 # print "%s" %(self.filenameList[i])
142
142
143 pathList = []
143 pathList = []
144
144
145 if not walk:
145 if not walk:
146 #pathList.append(path)
146 #pathList.append(path)
147 multi_path = path.split(',')
147 multi_path = path.split(',')
148 for single_path in multi_path:
148 for single_path in multi_path:
149 pathList.append(single_path)
149 pathList.append(single_path)
150
150
151 else:
151 else:
152 #dirList = []
152 #dirList = []
153 multi_path = path.split(',')
153 multi_path = path.split(',')
154 for single_path in multi_path:
154 for single_path in multi_path:
155 dirList = []
155 dirList = []
156 for thisPath in os.listdir(single_path):
156 for thisPath in os.listdir(single_path):
157 if not os.path.isdir(os.path.join(single_path,thisPath)):
157 if not os.path.isdir(os.path.join(single_path,thisPath)):
158 continue
158 continue
159 if not isDoyFolder(thisPath):
159 if not isDoyFolder(thisPath):
160 continue
160 continue
161
161
162 dirList.append(thisPath)
162 dirList.append(thisPath)
163
163
164 if not(dirList):
164 if not(dirList):
165 return None, None
165 return None, None
166
166
167 thisDate = startDate
167 thisDate = startDate
168
168
169 while(thisDate <= endDate):
169 while(thisDate <= endDate):
170 year = thisDate.timetuple().tm_year
170 year = thisDate.timetuple().tm_year
171 doy = thisDate.timetuple().tm_yday
171 doy = thisDate.timetuple().tm_yday
172
172
173 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
173 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
174 if len(matchlist) == 0:
174 if len(matchlist) == 0:
175 thisDate += datetime.timedelta(1)
175 thisDate += datetime.timedelta(1)
176 continue
176 continue
177 for match in matchlist:
177 for match in matchlist:
178 pathList.append(os.path.join(single_path,match))
178 pathList.append(os.path.join(single_path,match))
179
179
180 thisDate += datetime.timedelta(1)
180 thisDate += datetime.timedelta(1)
181
181
182 if pathList == []:
182 if pathList == []:
183 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
183 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
184 return None, None
184 return None, None
185
185
186 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
186 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
187
187
188 filenameList = []
188 filenameList = []
189 datetimeList = []
189 datetimeList = []
190 pathDict = {}
190 pathDict = {}
191 filenameList_to_sort = []
191 filenameList_to_sort = []
192
192
193 for i in range(len(pathList)):
193 for i in range(len(pathList)):
194
194
195 thisPath = pathList[i]
195 thisPath = pathList[i]
196
196
197 fileList = glob.glob1(thisPath, "*%s" %ext)
197 fileList = glob.glob1(thisPath, "*%s" %ext)
198 fileList.sort()
198 fileList.sort()
199 pathDict.setdefault(fileList[0])
199 pathDict.setdefault(fileList[0])
200 pathDict[fileList[0]] = i
200 pathDict[fileList[0]] = i
201 filenameList_to_sort.append(fileList[0])
201 filenameList_to_sort.append(fileList[0])
202
202
203 filenameList_to_sort.sort()
203 filenameList_to_sort.sort()
204
204
205 for file in filenameList_to_sort:
205 for file in filenameList_to_sort:
206 thisPath = pathList[pathDict[file]]
206 thisPath = pathList[pathDict[file]]
207
207
208 fileList = glob.glob1(thisPath, "*%s" %ext)
208 fileList = glob.glob1(thisPath, "*%s" %ext)
209 fileList.sort()
209 fileList.sort()
210
210
211 for file in fileList:
211 for file in fileList:
212
212
213 filename = os.path.join(thisPath,file)
213 filename = os.path.join(thisPath,file)
214 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
214 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
215
215
216 if not(thisDatetime):
216 if not(thisDatetime):
217 continue
217 continue
218
218
219 filenameList.append(filename)
219 filenameList.append(filename)
220 datetimeList.append(thisDatetime)
220 datetimeList.append(thisDatetime)
221
221
222 if not(filenameList):
222 if not(filenameList):
223 print "Any file was found for the time range %s - %s" %(startTime, endTime)
223 print "Any file was found for the time range %s - %s" %(startTime, endTime)
224 return None, None
224 return None, None
225
225
226 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
226 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
227 print
227 print
228
228
229 for i in range(len(filenameList)):
229 for i in range(len(filenameList)):
230 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
230 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
231
231
232 self.filenameList = filenameList
232 self.filenameList = filenameList
233 self.datetimeList = datetimeList
233 self.datetimeList = datetimeList
234
234
235 return pathList, filenameList
235 return pathList, filenameList
236
236
237 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
237 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
238 """
238 """
239 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
239 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
240
240
241 Inputs:
241 Inputs:
242 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
242 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
243
243
244 startTime : tiempo inicial del rango seleccionado en formato datetime.time
244 startTime : tiempo inicial del rango seleccionado en formato datetime.time
245
245
246 endTime : tiempo final del rango seleccionado en formato datetime.time
246 endTime : tiempo final del rango seleccionado en formato datetime.time
247
247
248 Return:
248 Return:
249 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
249 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
250 fecha especificado, de lo contrario retorna False.
250 fecha especificado, de lo contrario retorna False.
251
251
252 Excepciones:
252 Excepciones:
253 Si el archivo no existe o no puede ser abierto
253 Si el archivo no existe o no puede ser abierto
254 Si la cabecera no puede ser leida.
254 Si la cabecera no puede ser leida.
255
255
256 """
256 """
257
257
258 try:
258 try:
259 fp = fp = h5py.File(filename,'r')
259 fp = fp = h5py.File(filename,'r')
260 except IOError:
260 except IOError:
261 print "File %s can't be opened" %(filename)
261 traceback.print_exc()
262 return None
262 raise IOError, "The file %s can't be opened" %(filename)
263
263
264 grp = fp['Data']
264 grp = fp['Data']
265 timeAux = grp['time']
265 timeAux = grp['time']
266 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
266 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
267
267
268 fp.close()
268 fp.close()
269
269
270 if self.timezone == 'lt':
270 if self.timezone == 'lt':
271 time0 -= 5*3600
271 time0 -= 5*3600
272
272
273 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
273 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
274
274
275 if not (numpy.any(boolTimer)):
275 if not (numpy.any(boolTimer)):
276 return None
276 return None
277
277
278 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
278 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
279 return thisDatetime
279 return thisDatetime
280
280
281 def __checkPath(self):
281 def __checkPath(self):
282 if os.path.exists(self.path):
282 if os.path.exists(self.path):
283 self.status = 1
283 self.status = 1
284 else:
284 else:
285 self.status = 0
285 self.status = 0
286 print 'Path:%s does not exists'%self.path
286 print 'Path:%s does not exists'%self.path
287
287
288 return
288 return
289
289
290 def __setNextFileOffline(self):
290 def __setNextFileOffline(self):
291 idFile = self.fileIndex
291 idFile = self.fileIndex
292 idFile += 1
292 idFile += 1
293
293
294 if not(idFile < len(self.filenameList)):
294 if not(idFile < len(self.filenameList)):
295 print "No more Files"
295 print "No more Files"
296 return 0
296 return 0
297
297
298 filename = self.filenameList[idFile]
298 filename = self.filenameList[idFile]
299
299
300 filePointer = h5py.File(filename,'r')
300 filePointer = h5py.File(filename,'r')
301
301
302 self.flagIsNewFile = 1
302 self.flagIsNewFile = 1
303 self.fileIndex = idFile
303 self.fileIndex = idFile
304 self.filename = filename
304 self.filename = filename
305
305
306 self.fp = filePointer
306 self.fp = filePointer
307
307
308 print "Setting the file: %s"%self.filename
308 print "Setting the file: %s"%self.filename
309
309
310 self.__readMetadata()
310 self.__readMetadata()
311 self.__setBlockList()
311 self.__setBlockList()
312 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
312 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
313 self.nRecords = self.fp['Data'].attrs['nRecords']
313 self.nRecords = self.fp['Data'].attrs['nRecords']
314 self.blockIndex = 0
314 self.blockIndex = 0
315 return 1
315 return 1
316
316
317 def __setBlockList(self):
317 def __setBlockList(self):
318 '''
318 '''
319 self.fp
319 self.fp
320 self.startDateTime
320 self.startDateTime
321 self.endDateTime
321 self.endDateTime
322
322
323 self.blockList
323 self.blockList
324 self.blocksPerFile
324 self.blocksPerFile
325
325
326 '''
326 '''
327 filePointer = self.fp
327 filePointer = self.fp
328 secStart = self.secStart
328 secStart = self.secStart
329 secEnd = self.secEnd
329 secEnd = self.secEnd
330
330
331 grp = filePointer['Data']
331 grp = filePointer['Data']
332 timeVector = grp['time'].value.astype(numpy.float)[0]
332 timeVector = grp['time'].value.astype(numpy.float)[0]
333
333
334 if self.timezone == 'lt':
334 if self.timezone == 'lt':
335 timeVector -= 5*3600
335 timeVector -= 5*3600
336
336
337 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
337 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
338
338
339 self.blockList = ind
339 self.blockList = ind
340 self.blocksPerFile = len(ind)
340 self.blocksPerFile = len(ind)
341
341
342 return
342 return
343
343
344 def __readMetadata(self):
344 def __readMetadata(self):
345 '''
345 '''
346 self.pathMeta
346 self.pathMeta
347
347
348 self.listShapes
348 self.listShapes
349 self.listMetaname
349 self.listMetaname
350 self.listMeta
350 self.listMeta
351
351
352 '''
352 '''
353
353
354 grp = self.fp['Data']
354 grp = self.fp['Data']
355 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
355 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
356
356
357 if pathMeta == self.pathMeta:
357 if pathMeta == self.pathMeta:
358 return
358 return
359 else:
359 else:
360 self.pathMeta = pathMeta
360 self.pathMeta = pathMeta
361
361
362 filePointer = h5py.File(self.pathMeta,'r')
362 filePointer = h5py.File(self.pathMeta,'r')
363 groupPointer = filePointer['Metadata']
363 groupPointer = filePointer['Metadata']
364
364
365 listMetaname = []
365 listMetaname = []
366 listMetadata = []
366 listMetadata = []
367 for item in groupPointer.items():
367 for item in groupPointer.items():
368 name = item[0]
368 name = item[0]
369
369
370 if name=='array dimensions':
370 if name=='array dimensions':
371 table = groupPointer[name][:]
371 table = groupPointer[name][:]
372 listShapes = {}
372 listShapes = {}
373 for shapes in table:
373 for shapes in table:
374 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
374 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
375 else:
375 else:
376 data = groupPointer[name].value
376 data = groupPointer[name].value
377 listMetaname.append(name)
377 listMetaname.append(name)
378 listMetadata.append(data)
378 listMetadata.append(data)
379
379
380 if name=='type':
380 if name=='type':
381 self.__initDataOut(data)
381 self.__initDataOut(data)
382
382
383 filePointer.close()
383 filePointer.close()
384
384
385 self.listShapes = listShapes
385 self.listShapes = listShapes
386 self.listMetaname = listMetaname
386 self.listMetaname = listMetaname
387 self.listMeta = listMetadata
387 self.listMeta = listMetadata
388
388
389 return
389 return
390
390
391 def __readData(self):
391 def __readData(self):
392 grp = self.fp['Data']
392 grp = self.fp['Data']
393 listdataname = []
393 listdataname = []
394 listdata = []
394 listdata = []
395
395
396 for item in grp.items():
396 for item in grp.items():
397 name = item[0]
397 name = item[0]
398
398
399 if name == 'time':
399 if name == 'time':
400 listdataname.append('utctime')
400 listdataname.append('utctime')
401 timeAux = grp[name].value.astype(numpy.float)[0]
401 timeAux = grp[name].value.astype(numpy.float)[0]
402 listdata.append(timeAux)
402 listdata.append(timeAux)
403 continue
403 continue
404
404
405 listdataname.append(name)
405 listdataname.append(name)
406 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
406 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
407 listdata.append(array)
407 listdata.append(array)
408
408
409 self.listDataname = listdataname
409 self.listDataname = listdataname
410 self.listData = listdata
410 self.listData = listdata
411 return
411 return
412
412
413 def __setDataArray(self, nRecords, dataset, shapes):
413 def __setDataArray(self, nRecords, dataset, shapes):
414
414
415 nChannels = shapes[0] #Dimension 0
415 nChannels = shapes[0] #Dimension 0
416
416
417 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
417 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
418
418
419 nSamples = shapes[2] #Dimension 2, number of samples or ranges
419 nSamples = shapes[2] #Dimension 2, number of samples or ranges
420
420
421 mode = shapes[3]
421 mode = shapes[3]
422
422
423 # if nPoints>1:
423 # if nPoints>1:
424 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
424 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
425 # else:
425 # else:
426 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
426 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
427 #
427 #
428 # chn = 'channel'
428 # chn = 'channel'
429 #
429 #
430 # for i in range(nChannels):
430 # for i in range(nChannels):
431 #
431 #
432 # data = dataset[chn + str(i)].value
432 # data = dataset[chn + str(i)].value
433 #
433 #
434 # if nPoints>1:
434 # if nPoints>1:
435 # data = numpy.rollaxis(data,2)
435 # data = numpy.rollaxis(data,2)
436 #
436 #
437 # arrayData[:,i,:] = data
437 # arrayData[:,i,:] = data
438
438
439 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
439 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
440 doSqueeze = False
440 doSqueeze = False
441 if mode == 0:
441 if mode == 0:
442 strds = 'channel'
442 strds = 'channel'
443 nDatas = nChannels
443 nDatas = nChannels
444 newShapes = (nRecords,nPoints,nSamples)
444 newShapes = (nRecords,nPoints,nSamples)
445 if nPoints == 1:
445 if nPoints == 1:
446 doSqueeze = True
446 doSqueeze = True
447 axisSqueeze = 2
447 axisSqueeze = 2
448 else:
448 else:
449 strds = 'param'
449 strds = 'param'
450 nDatas = nPoints
450 nDatas = nPoints
451 newShapes = (nRecords,nChannels,nSamples)
451 newShapes = (nRecords,nChannels,nSamples)
452 if nChannels == 1:
452 if nChannels == 1:
453 doSqueeze = True
453 doSqueeze = True
454 axisSqueeze = 1
454 axisSqueeze = 1
455
455
456 for i in range(nDatas):
456 for i in range(nDatas):
457
457
458 data = dataset[strds + str(i)].value
458 data = dataset[strds + str(i)].value
459 data = data.reshape(newShapes)
459 data = data.reshape(newShapes)
460
460
461 if mode == 0:
461 if mode == 0:
462 arrayData[:,i,:,:] = data
462 arrayData[:,i,:,:] = data
463 else:
463 else:
464 arrayData[:,:,i,:] = data
464 arrayData[:,:,i,:] = data
465
465
466 if doSqueeze:
466 if doSqueeze:
467 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
467 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
468
468
469 return arrayData
469 return arrayData
470
470
471 def __initDataOut(self, type):
471 def __initDataOut(self, type):
472
472
473 # if type =='Parameters':
473 # if type =='Parameters':
474 # self.dataOut = Parameters()
474 # self.dataOut = Parameters()
475 # elif type =='Spectra':
475 # elif type =='Spectra':
476 # self.dataOut = Spectra()
476 # self.dataOut = Spectra()
477 # elif type =='Voltage':
477 # elif type =='Voltage':
478 # self.dataOut = Voltage()
478 # self.dataOut = Voltage()
479 # elif type =='Correlation':
479 # elif type =='Correlation':
480 # self.dataOut = Correlation()
480 # self.dataOut = Correlation()
481
481
482 return
482 return
483
483
484 def __setDataOut(self):
484 def __setDataOut(self):
485 listMeta = self.listMeta
485 listMeta = self.listMeta
486 listMetaname = self.listMetaname
486 listMetaname = self.listMetaname
487 listDataname = self.listDataname
487 listDataname = self.listDataname
488 listData = self.listData
488 listData = self.listData
489
489
490 blockIndex = self.blockIndex
490 blockIndex = self.blockIndex
491 blockList = self.blockList
491 blockList = self.blockList
492
492
493 for i in range(len(listMeta)):
493 for i in range(len(listMeta)):
494 setattr(self.dataOut,listMetaname[i],listMeta[i])
494 setattr(self.dataOut,listMetaname[i],listMeta[i])
495
495
496 for j in range(len(listData)):
496 for j in range(len(listData)):
497 if listDataname[j]=='utctime':
497 if listDataname[j]=='utctime':
498 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
498 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
499 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
499 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
500 continue
500 continue
501
501
502 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
502 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
503
503
504 return self.dataOut.data_param
504 return self.dataOut.data_param
505
505
506 def getData(self):
506 def getData(self):
507
507
508 # if self.flagNoMoreFiles:
508 # if self.flagNoMoreFiles:
509 # self.dataOut.flagNoData = True
509 # self.dataOut.flagNoData = True
510 # print 'Process finished'
510 # print 'Process finished'
511 # return 0
511 # return 0
512 #
512 #
513 if self.blockIndex==self.blocksPerFile:
513 if self.blockIndex==self.blocksPerFile:
514 if not( self.__setNextFileOffline() ):
514 if not( self.__setNextFileOffline() ):
515 self.dataOut.flagNoData = True
515 self.dataOut.flagNoData = True
516 return 0
516 return 0
517
517
518 #
518 #
519 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
519 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
520 # self.dataOut.flagNoData = True
520 # self.dataOut.flagNoData = True
521 # return 0
521 # return 0
522
522
523 self.__readData()
523 self.__readData()
524 self.__setDataOut()
524 self.__setDataOut()
525 self.dataOut.flagNoData = False
525 self.dataOut.flagNoData = False
526
526
527 self.blockIndex += 1
527 self.blockIndex += 1
528
528
529 return
529 return
530
530
531 def run(self, **kwargs):
531 def run(self, **kwargs):
532
532
533 if not(self.isConfig):
533 if not(self.isConfig):
534 self.setup(**kwargs)
534 self.setup(**kwargs)
535 # self.setObjProperties()
535 # self.setObjProperties()
536 self.isConfig = True
536 self.isConfig = True
537
537
538 self.getData()
538 self.getData()
539
539
540 return
540 return
541
541
542 class HDF5Writer(Operation):
542 class HDF5Writer(Operation):
543
543
544 ext = ".hdf5"
544 ext = ".hdf5"
545
545
546 optchar = "D"
546 optchar = "D"
547
547
548 metaoptchar = "M"
548 metaoptchar = "M"
549
549
550 metaFile = None
550 metaFile = None
551
551
552 filename = None
552 filename = None
553
553
554 path = None
554 path = None
555
555
556 setFile = None
556 setFile = None
557
557
558 fp = None
558 fp = None
559
559
560 grp = None
560 grp = None
561
561
562 ds = None
562 ds = None
563
563
564 firsttime = True
564 firsttime = True
565
565
566 #Configurations
566 #Configurations
567
567
568 blocksPerFile = None
568 blocksPerFile = None
569
569
570 blockIndex = None
570 blockIndex = None
571
571
572 dataOut = None
572 dataOut = None
573
573
574 #Data Arrays
574 #Data Arrays
575
575
576 dataList = None
576 dataList = None
577
577
578 metadataList = None
578 metadataList = None
579
579
580 arrayDim = None
580 arrayDim = None
581
581
582 tableDim = None
582 tableDim = None
583
583
584 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
584 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
585
585
586 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
586 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
587
587
588 mode = None
588 mode = None
589
589
590 nDatas = None #Number of datasets to be stored per array
590 nDatas = None #Number of datasets to be stored per array
591
591
592 nDims = None #Number Dimensions in each dataset
592 nDims = None #Number Dimensions in each dataset
593
593
594 nDimsForDs = None
594 nDimsForDs = None
595
595
596 currentDay = None
597
596 def __init__(self):
598 def __init__(self):
597
599
598 Operation.__init__(self)
600 Operation.__init__(self)
599 self.isConfig = False
601 self.isConfig = False
600 return
602 return
601
603
602
603 def setup(self, dataOut, **kwargs):
604 def setup(self, dataOut, **kwargs):
604
605
605 self.path = kwargs['path']
606 self.path = kwargs['path']
606
607
607 if kwargs.has_key('ext'):
608 if kwargs.has_key('ext'):
608 self.ext = kwargs['ext']
609 self.ext = kwargs['ext']
609
610
610 if kwargs.has_key('blocksPerFile'):
611 if kwargs.has_key('blocksPerFile'):
611 self.blocksPerFile = kwargs['blocksPerFile']
612 self.blocksPerFile = kwargs['blocksPerFile']
612 else:
613 else:
613 self.blocksPerFile = 10
614 self.blocksPerFile = 10
614
615
615 self.metadataList = kwargs['metadataList']
616 self.metadataList = kwargs['metadataList']
616
617
617 self.dataList = kwargs['dataList']
618 self.dataList = kwargs['dataList']
618
619
619 self.dataOut = dataOut
620 self.dataOut = dataOut
620
621
621 if kwargs.has_key('mode'):
622 if kwargs.has_key('mode'):
622 mode = kwargs['mode']
623 mode = kwargs['mode']
623
624
624 if type(mode) == int:
625 if type(mode) == int:
625 mode = numpy.zeros(len(self.dataList)) + mode
626 mode = numpy.zeros(len(self.dataList)) + mode
626 else:
627 else:
627 mode = numpy.zeros(len(self.dataList))
628 mode = numpy.zeros(len(self.dataList))
628
629
629 self.mode = mode
630 self.mode = mode
630
631
631 arrayDim = numpy.zeros((len(self.dataList),5))
632 arrayDim = numpy.zeros((len(self.dataList),5))
632
633
633 #Table dimensions
634 #Table dimensions
634
635
635 dtype0 = self.dtype
636 dtype0 = self.dtype
636
637
637 tableList = []
638 tableList = []
638
639
639 for i in range(len(self.dataList)):
640 for i in range(len(self.dataList)):
640
641
641 dataAux = getattr(self.dataOut, self.dataList[i])
642 dataAux = getattr(self.dataOut, self.dataList[i])
642
643
643 if type(dataAux)==float or type(dataAux)==int:
644 if type(dataAux)==float or type(dataAux)==int:
644 arrayDim[i,0] = 1
645 arrayDim[i,0] = 1
645 else:
646 else:
647
648 if dataAux == None:
649 return 0
650
646 arrayDim0 = dataAux.shape
651 arrayDim0 = dataAux.shape
647 arrayDim[i,0] = len(arrayDim0)
652 arrayDim[i,0] = len(arrayDim0)
648 arrayDim[i,4] = mode[i]
653 arrayDim[i,4] = mode[i]
649
654
650 if len(arrayDim0) == 3:
655 if len(arrayDim0) == 3:
651 arrayDim[i,1:-1] = numpy.array(arrayDim0)
656 arrayDim[i,1:-1] = numpy.array(arrayDim0)
652 elif len(arrayDim0) == 2:
657 elif len(arrayDim0) == 2:
653 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
658 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
654 elif len(arrayDim0) == 1:
659 elif len(arrayDim0) == 1:
655 arrayDim[i,3] = arrayDim0
660 arrayDim[i,3] = arrayDim0
656 elif len(arrayDim0) == 0:
661 elif len(arrayDim0) == 0:
657 arrayDim[i,0] = 1
662 arrayDim[i,0] = 1
658 arrayDim[i,3] = 1
663 arrayDim[i,3] = 1
659
664
660 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
665 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
661 tableList.append(table)
666 tableList.append(table)
662
667
663 self.arrayDim = arrayDim
668 self.arrayDim = arrayDim
664 self.tableDim = numpy.array(tableList, dtype = dtype0)
669 self.tableDim = numpy.array(tableList, dtype = dtype0)
665 self.blockIndex = 0
670 self.blockIndex = 0
666
671
667 return
672 timeTuple = time.localtime(dataOut.utctime)
673 self.currentDay = timeTuple.tm_yday
674 return 1
668
675
669 def putMetadata(self):
676 def putMetadata(self):
670
677
671 fp = self.createMetadataFile()
678 fp = self.createMetadataFile()
672 self.writeMetadata(fp)
679 self.writeMetadata(fp)
673 fp.close()
680 fp.close()
674 return
681 return
675
682
676 def createMetadataFile(self):
683 def createMetadataFile(self):
677 ext = self.ext
684 ext = self.ext
678 path = self.path
685 path = self.path
679 setFile = self.setFile
686 setFile = self.setFile
680
687
681 timeTuple = time.localtime(self.dataOut.utctime)
688 timeTuple = time.localtime(self.dataOut.utctime)
682
689
683 subfolder = ''
690 subfolder = ''
684 fullpath = os.path.join( path, subfolder )
691 fullpath = os.path.join( path, subfolder )
685
692
686 if not( os.path.exists(fullpath) ):
693 if not( os.path.exists(fullpath) ):
687 os.mkdir(fullpath)
694 os.mkdir(fullpath)
688 setFile = -1 #inicializo mi contador de seteo
695 setFile = -1 #inicializo mi contador de seteo
689
696
690 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
697 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
691 fullpath = os.path.join( path, subfolder )
698 fullpath = os.path.join( path, subfolder )
692
699
693 if not( os.path.exists(fullpath) ):
700 if not( os.path.exists(fullpath) ):
694 os.mkdir(fullpath)
701 os.mkdir(fullpath)
695 setFile = -1 #inicializo mi contador de seteo
702 setFile = -1 #inicializo mi contador de seteo
696
703
697 else:
704 else:
698 filesList = os.listdir( fullpath )
705 filesList = os.listdir( fullpath )
699 filesList = sorted( filesList, key=str.lower )
706 filesList = sorted( filesList, key=str.lower )
700 if len( filesList ) > 0:
707 if len( filesList ) > 0:
701 filesList = [k for k in filesList if 'M' in k]
708 filesList = [k for k in filesList if 'M' in k]
702 filen = filesList[-1]
709 filen = filesList[-1]
703 # el filename debera tener el siguiente formato
710 # el filename debera tener el siguiente formato
704 # 0 1234 567 89A BCDE (hex)
711 # 0 1234 567 89A BCDE (hex)
705 # x YYYY DDD SSS .ext
712 # x YYYY DDD SSS .ext
706 if isNumber( filen[8:11] ):
713 if isNumber( filen[8:11] ):
707 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
714 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
708 else:
715 else:
709 setFile = -1
716 setFile = -1
710 else:
717 else:
711 setFile = -1 #inicializo mi contador de seteo
718 setFile = -1 #inicializo mi contador de seteo
712
719
713 setFile += 1
720 setFile += 1
714
721
715 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
722 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
716 timeTuple.tm_year,
723 timeTuple.tm_year,
717 timeTuple.tm_yday,
724 timeTuple.tm_yday,
718 setFile,
725 setFile,
719 ext )
726 ext )
720
727
721 filename = os.path.join( path, subfolder, file )
728 filename = os.path.join( path, subfolder, file )
722 self.metaFile = file
729 self.metaFile = file
723 #Setting HDF5 File
730 #Setting HDF5 File
724 fp = h5py.File(filename,'w')
731 fp = h5py.File(filename,'w')
725
732
726 return fp
733 return fp
727
734
728 def writeMetadata(self, fp):
735 def writeMetadata(self, fp):
729
736
730 grp = fp.create_group("Metadata")
737 grp = fp.create_group("Metadata")
731 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
738 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
732
739
733 for i in range(len(self.metadataList)):
740 for i in range(len(self.metadataList)):
734 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
741 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
735 return
742 return
736
743
744 def dateFlag(self):
745
746 timeTuple = time.localtime(self.dataOut.utctime)
747 dataDay = timeTuple.tm_yday
748
749 if dataDay == self.currentDay:
750 return False
751
752 self.currentDay = dataDay
753 return True
754
737 def setNextFile(self):
755 def setNextFile(self):
738
756
739 ext = self.ext
757 ext = self.ext
740 path = self.path
758 path = self.path
741 setFile = self.setFile
759 setFile = self.setFile
742 mode = self.mode
760 mode = self.mode
743
761
744 timeTuple = time.localtime(self.dataOut.utctime)
762 timeTuple = time.localtime(self.dataOut.utctime)
745 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
763 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
746
764
747 fullpath = os.path.join( path, subfolder )
765 fullpath = os.path.join( path, subfolder )
748
766
749 if os.path.exists(fullpath):
767 if os.path.exists(fullpath):
750 filesList = os.listdir( fullpath )
768 filesList = os.listdir( fullpath )
751 filesList = [k for k in filesList if 'D' in k]
769 filesList = [k for k in filesList if 'D' in k]
752 if len( filesList ) > 0:
770 if len( filesList ) > 0:
753 filesList = sorted( filesList, key=str.lower )
771 filesList = sorted( filesList, key=str.lower )
754 filen = filesList[-1]
772 filen = filesList[-1]
755 # el filename debera tener el siguiente formato
773 # el filename debera tener el siguiente formato
756 # 0 1234 567 89A BCDE (hex)
774 # 0 1234 567 89A BCDE (hex)
757 # x YYYY DDD SSS .ext
775 # x YYYY DDD SSS .ext
758 if isNumber( filen[8:11] ):
776 if isNumber( filen[8:11] ):
759 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
777 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
760 else:
778 else:
761 setFile = -1
779 setFile = -1
762 else:
780 else:
763 setFile = -1 #inicializo mi contador de seteo
781 setFile = -1 #inicializo mi contador de seteo
764
782 else:
783 os.mkdir(fullpath)
784 setFile = -1 #inicializo mi contador de seteo
785
765 setFile += 1
786 setFile += 1
766
787
767 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
788 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
768 timeTuple.tm_year,
789 timeTuple.tm_year,
769 timeTuple.tm_yday,
790 timeTuple.tm_yday,
770 setFile,
791 setFile,
771 ext )
792 ext )
772
793
773 filename = os.path.join( path, subfolder, file )
794 filename = os.path.join( path, subfolder, file )
774
795
775 #Setting HDF5 File
796 #Setting HDF5 File
776 fp = h5py.File(filename,'w')
797 fp = h5py.File(filename,'w')
798
799 #writemetadata
800 self.writeMetadata(fp)
801
777 grp = fp.create_group("Data")
802 grp = fp.create_group("Data")
778 grp.attrs['metadata'] = self.metaFile
803 # grp.attrs['metadata'] = self.metaFile
779
804
780 # grp.attrs['blocksPerFile'] = 0
805 # grp.attrs['blocksPerFile'] = 0
781
806
782 ds = []
807 ds = []
783 data = []
808 data = []
784 nDimsForDs = []
809 nDimsForDs = []
785
810
786 nDatas = numpy.zeros(len(self.dataList))
811 nDatas = numpy.zeros(len(self.dataList))
787 nDims = self.arrayDim[:,0]
812 nDims = self.arrayDim[:,0]
788
813
789 nDim1 = self.arrayDim[:,2]
814 nDim1 = self.arrayDim[:,2]
790 nDim0 = self.arrayDim[:,3]
815 nDim0 = self.arrayDim[:,3]
791
816
792 for i in range(len(self.dataList)):
817 for i in range(len(self.dataList)):
793
818
794 if nDims[i]==1:
819 if nDims[i]==1:
795 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
820 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
796 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
821 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
797 ds.append(ds0)
822 ds.append(ds0)
798 data.append([])
823 data.append([])
799 nDimsForDs.append(nDims[i])
824 nDimsForDs.append(nDims[i])
800 else:
825 else:
801
826
802 if mode[i]==0:
827 if mode[i]==0:
803 strMode = "channel"
828 strMode = "channel"
804 nDatas[i] = self.arrayDim[i,1]
829 nDatas[i] = self.arrayDim[i,1]
805 else:
830 else:
806 strMode = "param"
831 strMode = "param"
807 nDatas[i] = self.arrayDim[i,2]
832 nDatas[i] = self.arrayDim[i,2]
808
833
809 if nDims[i]==2:
834 if nDims[i]==2:
810 nDatas[i] = self.arrayDim[i,2]
835 nDatas[i] = self.arrayDim[i,2]
811
836
812 grp0 = grp.create_group(self.dataList[i])
837 grp0 = grp.create_group(self.dataList[i])
813
838
814 for j in range(int(nDatas[i])):
839 for j in range(int(nDatas[i])):
815 tableName = strMode + str(j)
840 tableName = strMode + str(j)
816
841
817 if nDims[i] == 3:
842 if nDims[i] == 3:
818 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
843 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
819 else:
844 else:
820 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
845 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
821
846
822 ds.append(ds0)
847 ds.append(ds0)
823 data.append([])
848 data.append([])
824 nDimsForDs.append(nDims[i])
849 nDimsForDs.append(nDims[i])
825 self.nDatas = nDatas
850 self.nDatas = nDatas
826 self.nDims = nDims
851 self.nDims = nDims
827 self.nDimsForDs = nDimsForDs
852 self.nDimsForDs = nDimsForDs
828 #Saving variables
853 #Saving variables
829 print 'Writing the file: %s'%filename
854 print 'Writing the file: %s'%filename
830 self.filename = filename
855 self.filename = filename
831 self.fp = fp
856 self.fp = fp
832 self.grp = grp
857 self.grp = grp
833 self.grp.attrs.modify('nRecords', 1)
858 self.grp.attrs.modify('nRecords', 1)
834 self.ds = ds
859 self.ds = ds
835 self.data = data
860 self.data = data
836
861
837 self.setFile = setFile
862 self.setFile = setFile
838 self.firsttime = True
863 self.firsttime = True
839 self.blockIndex = 0
864 self.blockIndex = 0
840 return
865 return
841
866
842 def putData(self):
867 def putData(self):
843
868
844 if not self.firsttime:
869 if not self.firsttime:
845 self.readBlock()
870 self.readBlock()
846
871
847 if self.blockIndex == self.blocksPerFile:
872 if self.blockIndex == self.blocksPerFile or self.dateFlag():
848
873
849 self.setNextFile()
874 self.setNextFile()
850
875
851 self.setBlock()
876 self.setBlock()
852 self.writeBlock()
877 self.writeBlock()
853
878
854 self.fp.flush()
879 self.fp.flush()
855 self.fp.close()
880 self.fp.close()
856
881
857 return
882 return
858
883
859 def readBlock(self):
884 def readBlock(self):
860
885
861 '''
886 '''
862 data Array configured
887 data Array configured
863
888
864
889
865 self.data
890 self.data
866 '''
891 '''
867 ds = self.ds
892 ds = self.ds
868 #Setting HDF5 File
893 #Setting HDF5 File
869 fp = h5py.File(self.filename,'r+')
894 fp = h5py.File(self.filename,'r+')
870 grp = fp["Data"]
895 grp = fp["Data"]
871 ind = 0
896 ind = 0
872
897
873 # grp.attrs['blocksPerFile'] = 0
898 # grp.attrs['blocksPerFile'] = 0
874 for i in range(len(self.dataList)):
899 for i in range(len(self.dataList)):
875
900
876 if self.nDims[i]==1:
901 if self.nDims[i]==1:
877 ds0 = grp[self.dataList[i]]
902 ds0 = grp[self.dataList[i]]
878 ds[ind] = ds0
903 ds[ind] = ds0
879 ind += 1
904 ind += 1
880 else:
905 else:
881 if self.mode[i]==0:
906 if self.mode[i]==0:
882 strMode = "channel"
907 strMode = "channel"
883 else:
908 else:
884 strMode = "param"
909 strMode = "param"
885
910
886 grp0 = grp[self.dataList[i]]
911 grp0 = grp[self.dataList[i]]
887
912
888 for j in range(int(self.nDatas[i])):
913 for j in range(int(self.nDatas[i])):
889 tableName = strMode + str(j)
914 tableName = strMode + str(j)
890 ds0 = grp0[tableName]
915 ds0 = grp0[tableName]
891 ds[ind] = ds0
916 ds[ind] = ds0
892 ind += 1
917 ind += 1
893
918
894
919
895 self.fp = fp
920 self.fp = fp
896 self.grp = grp
921 self.grp = grp
897 self.ds = ds
922 self.ds = ds
898
923
899 return
924 return
900
925
901
902 def setBlock(self):
926 def setBlock(self):
903 '''
927 '''
904 data Array configured
928 data Array configured
905
929
906
930
907 self.data
931 self.data
908 '''
932 '''
909 #Creating Arrays
933 #Creating Arrays
910 data = self.data
934 data = self.data
911 nDatas = self.nDatas
935 nDatas = self.nDatas
912 nDims = self.nDims
936 nDims = self.nDims
913 mode = self.mode
937 mode = self.mode
914 ind = 0
938 ind = 0
915
939
916 for i in range(len(self.dataList)):
940 for i in range(len(self.dataList)):
917 dataAux = getattr(self.dataOut,self.dataList[i])
941 dataAux = getattr(self.dataOut,self.dataList[i])
918
942
919 if nDims[i] == 1:
943 if nDims[i] == 1:
920 # data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
944 # data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
921 data[ind] = dataAux
945 data[ind] = dataAux
922 # if not self.firsttime:
946 # if not self.firsttime:
923 # data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
947 # data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
924 ind += 1
948 ind += 1
925 else:
949 else:
926 for j in range(int(nDatas[i])):
950 for j in range(int(nDatas[i])):
927 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
951 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
928 data[ind] = dataAux[j,:]
952 data[ind] = dataAux[j,:]
929 else:
953 else:
930 data[ind] = dataAux[:,j,:]
954 data[ind] = dataAux[:,j,:]
931
955
932 # if nDims[i] == 3:
956 # if nDims[i] == 3:
933 # data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
957 # data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
934
958
935 # if not self.firsttime:
959 # if not self.firsttime:
936 # data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
960 # data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
937
961
938 # else:
962 # else:
939 # data[ind] = data[ind].reshape((1,data[ind].shape[0]))
963 # data[ind] = data[ind].reshape((1,data[ind].shape[0]))
940
964
941 # if not self.firsttime:
965 # if not self.firsttime:
942 # data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
966 # data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
943 ind += 1
967 ind += 1
944
968
945 self.data = data
969 self.data = data
946 return
970 return
947
971
948 def writeBlock(self):
972 def writeBlock(self):
949 '''
973 '''
950 Saves the block in the HDF5 file
974 Saves the block in the HDF5 file
951 '''
975 '''
952 for i in range(len(self.ds)):
976 for i in range(len(self.ds)):
953 if self.firsttime:
977 if self.firsttime:
954 # self.ds[i].resize(self.data[i].shape)
978 # self.ds[i].resize(self.data[i].shape)
955 # self.ds[i][self.blockIndex,:] = self.data[i]
979 # self.ds[i][self.blockIndex,:] = self.data[i]
956 if type(self.data[i]) == numpy.ndarray:
980 if type(self.data[i]) == numpy.ndarray:
957 nDims1 = len(self.ds[i].shape)
981 nDims1 = len(self.ds[i].shape)
958
982
959 if nDims1 == 3:
983 if nDims1 == 3:
960 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
984 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
961
985
962 self.ds[i].resize(self.data[i].shape)
986 self.ds[i].resize(self.data[i].shape)
963 self.ds[i][:] = self.data[i]
987 self.ds[i][:] = self.data[i]
964 else:
988 else:
965 if self.nDimsForDs[i] == 1:
989 if self.nDimsForDs[i] == 1:
966 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
990 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
967 self.ds[i][0,-1] = self.data[i]
991 self.ds[i][0,-1] = self.data[i]
968 elif self.nDimsForDs[i] == 2:
992 elif self.nDimsForDs[i] == 2:
969 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
993 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
970 self.ds[i][self.blockIndex,:] = self.data[i]
994 self.ds[i][self.blockIndex,:] = self.data[i]
971 elif self.nDimsForDs[i] == 3:
995 elif self.nDimsForDs[i] == 3:
972
996
973 dataShape = self.data[i].shape
997 dataShape = self.data[i].shape
974 dsShape = self.ds[i].shape
998 dsShape = self.ds[i].shape
975
999
976 if dataShape[0]==dsShape[0]:
1000 if dataShape[0]==dsShape[0]:
977 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1001 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
978 self.ds[i][:,:,-1] = self.data[i]
1002 self.ds[i][:,:,-1] = self.data[i]
979 else:
1003 else:
980 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2]))
1004 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2]))
981 self.ds[i][dsShape[0]:,:,0] = self.data[i]
1005 self.ds[i][dsShape[0]:,:,0] = self.data[i]
982 # self.ds[i].append(self.data[i])
983 # self.fp.flush()
984 # if not self.firsttime:
985 # self.fp.root.Data._v_attrs.nRecords = self.blockIndex
986
987 # if self.firsttime:
988 # self.fp.close()
989 # self.readBlock2()
990
1006
991 self.blockIndex += 1
1007 self.blockIndex += 1
992 self.firsttime = False
1008 self.firsttime = False
993 return
1009 return
994
1010
995 def run(self, dataOut, **kwargs):
1011 def run(self, dataOut, **kwargs):
1012
996 if not(self.isConfig):
1013 if not(self.isConfig):
997 self.setup(dataOut, **kwargs)
1014 flagdata = self.setup(dataOut, **kwargs)
1015
1016 if not(flagdata):
1017 return
1018
998 self.isConfig = True
1019 self.isConfig = True
999 self.putMetadata()
1020 # self.putMetadata()
1000 self.setNextFile()
1021 self.setNextFile()
1001
1022
1002 self.putData()
1023 self.putData()
1003 return
1024 return
1004
1025
1026
General Comments 0
You need to be logged in to leave comments. Login now