##// END OF EJS Templates
schain
RSS
tables was removed from imports
Miguel Valdez -
r639:87917d733682
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1005 +1,1004
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import tables
7
6
8 from schainpy.model.data.jrodata import *
7 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
8 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.io.jroIO_base import *
9 from schainpy.model.io.jroIO_base import *
11
10
12
11
13 class HDF5Reader(ProcessingUnit):
12 class HDF5Reader(ProcessingUnit):
14
13
15 ext = ".hdf5"
14 ext = ".hdf5"
16
15
17 optchar = "D"
16 optchar = "D"
18
17
19 timezone = None
18 timezone = None
20
19
21 secStart = None
20 secStart = None
22
21
23 secEnd = None
22 secEnd = None
24
23
25 fileIndex = None
24 fileIndex = None
26
25
27 blockIndex = None
26 blockIndex = None
28
27
29 blocksPerFile = None
28 blocksPerFile = None
30
29
31 path = None
30 path = None
32
31
33 #List of Files
32 #List of Files
34
33
35 filenameList = None
34 filenameList = None
36
35
37 datetimeList = None
36 datetimeList = None
38
37
39 #Hdf5 File
38 #Hdf5 File
40
39
41 fpMetadata = None
40 fpMetadata = None
42
41
43 pathMeta = None
42 pathMeta = None
44
43
45 listMetaname = None
44 listMetaname = None
46
45
47 listMeta = None
46 listMeta = None
48
47
49 listDataname = None
48 listDataname = None
50
49
51 listData = None
50 listData = None
52
51
53 listShapes = None
52 listShapes = None
54
53
55 fp = None
54 fp = None
56
55
57 #dataOut reconstruction
56 #dataOut reconstruction
58
57
59 dataOut = None
58 dataOut = None
60
59
61 nRecords = None
60 nRecords = None
62
61
63
62
64 def __init__(self):
63 def __init__(self):
65 self.dataOut = self.__createObjByDefault()
64 self.dataOut = self.__createObjByDefault()
66 return
65 return
67
66
68 def __createObjByDefault(self):
67 def __createObjByDefault(self):
69
68
70 dataObj = Parameters()
69 dataObj = Parameters()
71
70
72 return dataObj
71 return dataObj
73
72
74 def setup(self,path=None,
73 def setup(self,path=None,
75 startDate=None,
74 startDate=None,
76 endDate=None,
75 endDate=None,
77 startTime=datetime.time(0,0,0),
76 startTime=datetime.time(0,0,0),
78 endTime=datetime.time(23,59,59),
77 endTime=datetime.time(23,59,59),
79 walk=True,
78 walk=True,
80 timezone='ut',
79 timezone='ut',
81 all=0,
80 all=0,
82 online=False,
81 online=False,
83 ext=None):
82 ext=None):
84
83
85 if ext==None:
84 if ext==None:
86 ext = self.ext
85 ext = self.ext
87 self.timezone = timezone
86 self.timezone = timezone
88 # self.all = all
87 # self.all = all
89 # self.online = online
88 # self.online = online
90 self.path = path
89 self.path = path
91
90
92 startDateTime = datetime.datetime.combine(startDate,startTime)
91 startDateTime = datetime.datetime.combine(startDate,startTime)
93 endDateTime = datetime.datetime.combine(endDate,endTime)
92 endDateTime = datetime.datetime.combine(endDate,endTime)
94 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
93 secStart = (startDateTime-datetime.datetime(1970,1,1)).total_seconds()
95 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
94 secEnd = (endDateTime-datetime.datetime(1970,1,1)).total_seconds()
96
95
97 self.secStart = secStart
96 self.secStart = secStart
98 self.secEnd = secEnd
97 self.secEnd = secEnd
99
98
100 if not(online):
99 if not(online):
101 #Busqueda de archivos offline
100 #Busqueda de archivos offline
102 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
101 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, secStart, secEnd, walk)
103 else:
102 else:
104 self.__searchFilesOnline(path, walk)
103 self.__searchFilesOnline(path, walk)
105
104
106 if not(self.filenameList):
105 if not(self.filenameList):
107 print "There is no files into the folder: %s"%(path)
106 print "There is no files into the folder: %s"%(path)
108 sys.exit(-1)
107 sys.exit(-1)
109
108
110 # self.__getExpParameters()
109 # self.__getExpParameters()
111
110
112 self.fileIndex = -1
111 self.fileIndex = -1
113
112
114 self.__setNextFileOffline()
113 self.__setNextFileOffline()
115
114
116 self.__readMetadata()
115 self.__readMetadata()
117
116
118 self.blockIndex = 0
117 self.blockIndex = 0
119
118
120 return
119 return
121
120
122 def __searchFilesOffline(self,
121 def __searchFilesOffline(self,
123 path,
122 path,
124 startDate,
123 startDate,
125 endDate,
124 endDate,
126 ext,
125 ext,
127 startTime=datetime.time(0,0,0),
126 startTime=datetime.time(0,0,0),
128 endTime=datetime.time(23,59,59),
127 endTime=datetime.time(23,59,59),
129 secStart = 0,
128 secStart = 0,
130 secEnd = numpy.inf,
129 secEnd = numpy.inf,
131 walk=True):
130 walk=True):
132
131
133 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
132 # self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
134 #
133 #
135 # self.__checkPath()
134 # self.__checkPath()
136 #
135 #
137 # self.__findDataForDates()
136 # self.__findDataForDates()
138 #
137 #
139 # self.__selectDataForTimes()
138 # self.__selectDataForTimes()
140 #
139 #
141 # for i in range(len(self.filenameList)):
140 # for i in range(len(self.filenameList)):
142 # print "%s" %(self.filenameList[i])
141 # print "%s" %(self.filenameList[i])
143
142
144 pathList = []
143 pathList = []
145
144
146 if not walk:
145 if not walk:
147 #pathList.append(path)
146 #pathList.append(path)
148 multi_path = path.split(',')
147 multi_path = path.split(',')
149 for single_path in multi_path:
148 for single_path in multi_path:
150 pathList.append(single_path)
149 pathList.append(single_path)
151
150
152 else:
151 else:
153 #dirList = []
152 #dirList = []
154 multi_path = path.split(',')
153 multi_path = path.split(',')
155 for single_path in multi_path:
154 for single_path in multi_path:
156 dirList = []
155 dirList = []
157 for thisPath in os.listdir(single_path):
156 for thisPath in os.listdir(single_path):
158 if not os.path.isdir(os.path.join(single_path,thisPath)):
157 if not os.path.isdir(os.path.join(single_path,thisPath)):
159 continue
158 continue
160 if not isDoyFolder(thisPath):
159 if not isDoyFolder(thisPath):
161 continue
160 continue
162
161
163 dirList.append(thisPath)
162 dirList.append(thisPath)
164
163
165 if not(dirList):
164 if not(dirList):
166 return None, None
165 return None, None
167
166
168 thisDate = startDate
167 thisDate = startDate
169
168
170 while(thisDate <= endDate):
169 while(thisDate <= endDate):
171 year = thisDate.timetuple().tm_year
170 year = thisDate.timetuple().tm_year
172 doy = thisDate.timetuple().tm_yday
171 doy = thisDate.timetuple().tm_yday
173
172
174 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
173 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
175 if len(matchlist) == 0:
174 if len(matchlist) == 0:
176 thisDate += datetime.timedelta(1)
175 thisDate += datetime.timedelta(1)
177 continue
176 continue
178 for match in matchlist:
177 for match in matchlist:
179 pathList.append(os.path.join(single_path,match))
178 pathList.append(os.path.join(single_path,match))
180
179
181 thisDate += datetime.timedelta(1)
180 thisDate += datetime.timedelta(1)
182
181
183 if pathList == []:
182 if pathList == []:
184 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
183 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
185 return None, None
184 return None, None
186
185
187 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
186 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
188
187
189 filenameList = []
188 filenameList = []
190 datetimeList = []
189 datetimeList = []
191 pathDict = {}
190 pathDict = {}
192 filenameList_to_sort = []
191 filenameList_to_sort = []
193
192
194 for i in range(len(pathList)):
193 for i in range(len(pathList)):
195
194
196 thisPath = pathList[i]
195 thisPath = pathList[i]
197
196
198 fileList = glob.glob1(thisPath, "*%s" %ext)
197 fileList = glob.glob1(thisPath, "*%s" %ext)
199 fileList.sort()
198 fileList.sort()
200 pathDict.setdefault(fileList[0])
199 pathDict.setdefault(fileList[0])
201 pathDict[fileList[0]] = i
200 pathDict[fileList[0]] = i
202 filenameList_to_sort.append(fileList[0])
201 filenameList_to_sort.append(fileList[0])
203
202
204 filenameList_to_sort.sort()
203 filenameList_to_sort.sort()
205
204
206 for file in filenameList_to_sort:
205 for file in filenameList_to_sort:
207 thisPath = pathList[pathDict[file]]
206 thisPath = pathList[pathDict[file]]
208
207
209 fileList = glob.glob1(thisPath, "*%s" %ext)
208 fileList = glob.glob1(thisPath, "*%s" %ext)
210 fileList.sort()
209 fileList.sort()
211
210
212 for file in fileList:
211 for file in fileList:
213
212
214 filename = os.path.join(thisPath,file)
213 filename = os.path.join(thisPath,file)
215 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
214 thisDatetime = self.__isFileinThisTime(filename, secStart, secEnd)
216
215
217 if not(thisDatetime):
216 if not(thisDatetime):
218 continue
217 continue
219
218
220 filenameList.append(filename)
219 filenameList.append(filename)
221 datetimeList.append(thisDatetime)
220 datetimeList.append(thisDatetime)
222
221
223 if not(filenameList):
222 if not(filenameList):
224 print "Any file was found for the time range %s - %s" %(startTime, endTime)
223 print "Any file was found for the time range %s - %s" %(startTime, endTime)
225 return None, None
224 return None, None
226
225
227 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
226 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
228 print
227 print
229
228
230 for i in range(len(filenameList)):
229 for i in range(len(filenameList)):
231 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
230 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
232
231
233 self.filenameList = filenameList
232 self.filenameList = filenameList
234 self.datetimeList = datetimeList
233 self.datetimeList = datetimeList
235
234
236 return pathList, filenameList
235 return pathList, filenameList
237
236
238 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
237 def __isFileinThisTime(self, filename, startSeconds, endSeconds):
239 """
238 """
240 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
239 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
241
240
242 Inputs:
241 Inputs:
243 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
242 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
244
243
245 startTime : tiempo inicial del rango seleccionado en formato datetime.time
244 startTime : tiempo inicial del rango seleccionado en formato datetime.time
246
245
247 endTime : tiempo final del rango seleccionado en formato datetime.time
246 endTime : tiempo final del rango seleccionado en formato datetime.time
248
247
249 Return:
248 Return:
250 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
249 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
251 fecha especificado, de lo contrario retorna False.
250 fecha especificado, de lo contrario retorna False.
252
251
253 Excepciones:
252 Excepciones:
254 Si el archivo no existe o no puede ser abierto
253 Si el archivo no existe o no puede ser abierto
255 Si la cabecera no puede ser leida.
254 Si la cabecera no puede ser leida.
256
255
257 """
256 """
258
257
259 try:
258 try:
260 fp = fp = h5py.File(filename,'r')
259 fp = fp = h5py.File(filename,'r')
261 except IOError:
260 except IOError:
262 traceback.print_exc()
261 traceback.print_exc()
263 raise IOError, "The file %s can't be opened" %(filename)
262 raise IOError, "The file %s can't be opened" %(filename)
264
263
265 grp = fp['Data']
264 grp = fp['Data']
266 timeAux = grp['time']
265 timeAux = grp['time']
267 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
266 time0 = timeAux[:][0].astype(numpy.float) #Time Vector
268
267
269 fp.close()
268 fp.close()
270
269
271 if self.timezone == 'lt':
270 if self.timezone == 'lt':
272 time0 -= 5*3600
271 time0 -= 5*3600
273
272
274 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
273 boolTimer = numpy.logical_and(time0 >= startSeconds,time0 < endSeconds)
275
274
276 if not (numpy.any(boolTimer)):
275 if not (numpy.any(boolTimer)):
277 return None
276 return None
278
277
279 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
278 thisDatetime = datetime.datetime.utcfromtimestamp(time0[0])
280 return thisDatetime
279 return thisDatetime
281
280
282 def __checkPath(self):
281 def __checkPath(self):
283 if os.path.exists(self.path):
282 if os.path.exists(self.path):
284 self.status = 1
283 self.status = 1
285 else:
284 else:
286 self.status = 0
285 self.status = 0
287 print 'Path:%s does not exists'%self.path
286 print 'Path:%s does not exists'%self.path
288
287
289 return
288 return
290
289
291 def __setNextFileOffline(self):
290 def __setNextFileOffline(self):
292 idFile = self.fileIndex
291 idFile = self.fileIndex
293 idFile += 1
292 idFile += 1
294
293
295 if not(idFile < len(self.filenameList)):
294 if not(idFile < len(self.filenameList)):
296 print "No more Files"
295 print "No more Files"
297 return 0
296 return 0
298
297
299 filename = self.filenameList[idFile]
298 filename = self.filenameList[idFile]
300
299
301 filePointer = h5py.File(filename,'r')
300 filePointer = h5py.File(filename,'r')
302
301
303 self.flagIsNewFile = 1
302 self.flagIsNewFile = 1
304 self.fileIndex = idFile
303 self.fileIndex = idFile
305 self.filename = filename
304 self.filename = filename
306
305
307 self.fp = filePointer
306 self.fp = filePointer
308
307
309 print "Setting the file: %s"%self.filename
308 print "Setting the file: %s"%self.filename
310
309
311 self.__readMetadata()
310 self.__readMetadata()
312 self.__setBlockList()
311 self.__setBlockList()
313 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
312 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
314 self.nRecords = self.fp['Data'].attrs['nRecords']
313 self.nRecords = self.fp['Data'].attrs['nRecords']
315 self.blockIndex = 0
314 self.blockIndex = 0
316 return 1
315 return 1
317
316
318 def __setBlockList(self):
317 def __setBlockList(self):
319 '''
318 '''
320 self.fp
319 self.fp
321 self.startDateTime
320 self.startDateTime
322 self.endDateTime
321 self.endDateTime
323
322
324 self.blockList
323 self.blockList
325 self.blocksPerFile
324 self.blocksPerFile
326
325
327 '''
326 '''
328 filePointer = self.fp
327 filePointer = self.fp
329 secStart = self.secStart
328 secStart = self.secStart
330 secEnd = self.secEnd
329 secEnd = self.secEnd
331
330
332 grp = filePointer['Data']
331 grp = filePointer['Data']
333 timeVector = grp['time'].value.astype(numpy.float)[0]
332 timeVector = grp['time'].value.astype(numpy.float)[0]
334
333
335 if self.timezone == 'lt':
334 if self.timezone == 'lt':
336 timeVector -= 5*3600
335 timeVector -= 5*3600
337
336
338 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
337 ind = numpy.where(numpy.logical_and(timeVector >= secStart , timeVector < secEnd))[0]
339
338
340 self.blockList = ind
339 self.blockList = ind
341 self.blocksPerFile = len(ind)
340 self.blocksPerFile = len(ind)
342
341
343 return
342 return
344
343
345 def __readMetadata(self):
344 def __readMetadata(self):
346 '''
345 '''
347 self.pathMeta
346 self.pathMeta
348
347
349 self.listShapes
348 self.listShapes
350 self.listMetaname
349 self.listMetaname
351 self.listMeta
350 self.listMeta
352
351
353 '''
352 '''
354
353
355 grp = self.fp['Data']
354 grp = self.fp['Data']
356 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
355 pathMeta = os.path.join(self.path, grp.attrs['metadata'])
357
356
358 if pathMeta == self.pathMeta:
357 if pathMeta == self.pathMeta:
359 return
358 return
360 else:
359 else:
361 self.pathMeta = pathMeta
360 self.pathMeta = pathMeta
362
361
363 filePointer = h5py.File(self.pathMeta,'r')
362 filePointer = h5py.File(self.pathMeta,'r')
364 groupPointer = filePointer['Metadata']
363 groupPointer = filePointer['Metadata']
365
364
366 listMetaname = []
365 listMetaname = []
367 listMetadata = []
366 listMetadata = []
368 for item in groupPointer.items():
367 for item in groupPointer.items():
369 name = item[0]
368 name = item[0]
370
369
371 if name=='array dimensions':
370 if name=='array dimensions':
372 table = groupPointer[name][:]
371 table = groupPointer[name][:]
373 listShapes = {}
372 listShapes = {}
374 for shapes in table:
373 for shapes in table:
375 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
374 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4]])
376 else:
375 else:
377 data = groupPointer[name].value
376 data = groupPointer[name].value
378 listMetaname.append(name)
377 listMetaname.append(name)
379 listMetadata.append(data)
378 listMetadata.append(data)
380
379
381 if name=='type':
380 if name=='type':
382 self.__initDataOut(data)
381 self.__initDataOut(data)
383
382
384 filePointer.close()
383 filePointer.close()
385
384
386 self.listShapes = listShapes
385 self.listShapes = listShapes
387 self.listMetaname = listMetaname
386 self.listMetaname = listMetaname
388 self.listMeta = listMetadata
387 self.listMeta = listMetadata
389
388
390 return
389 return
391
390
392 def __readData(self):
391 def __readData(self):
393 grp = self.fp['Data']
392 grp = self.fp['Data']
394 listdataname = []
393 listdataname = []
395 listdata = []
394 listdata = []
396
395
397 for item in grp.items():
396 for item in grp.items():
398 name = item[0]
397 name = item[0]
399
398
400 if name == 'time':
399 if name == 'time':
401 listdataname.append('utctime')
400 listdataname.append('utctime')
402 timeAux = grp[name].value.astype(numpy.float)[0]
401 timeAux = grp[name].value.astype(numpy.float)[0]
403 listdata.append(timeAux)
402 listdata.append(timeAux)
404 continue
403 continue
405
404
406 listdataname.append(name)
405 listdataname.append(name)
407 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
406 array = self.__setDataArray(self.nRecords, grp[name],self.listShapes[name])
408 listdata.append(array)
407 listdata.append(array)
409
408
410 self.listDataname = listdataname
409 self.listDataname = listdataname
411 self.listData = listdata
410 self.listData = listdata
412 return
411 return
413
412
414 def __setDataArray(self, nRecords, dataset, shapes):
413 def __setDataArray(self, nRecords, dataset, shapes):
415
414
416 nChannels = shapes[0] #Dimension 0
415 nChannels = shapes[0] #Dimension 0
417
416
418 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
417 nPoints = shapes[1] #Dimension 1, number of Points or Parameters
419
418
420 nSamples = shapes[2] #Dimension 2, number of samples or ranges
419 nSamples = shapes[2] #Dimension 2, number of samples or ranges
421
420
422 mode = shapes[3]
421 mode = shapes[3]
423
422
424 # if nPoints>1:
423 # if nPoints>1:
425 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
424 # arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
426 # else:
425 # else:
427 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
426 # arrayData = numpy.zeros((nRecords,nChannels,nSamples))
428 #
427 #
429 # chn = 'channel'
428 # chn = 'channel'
430 #
429 #
431 # for i in range(nChannels):
430 # for i in range(nChannels):
432 #
431 #
433 # data = dataset[chn + str(i)].value
432 # data = dataset[chn + str(i)].value
434 #
433 #
435 # if nPoints>1:
434 # if nPoints>1:
436 # data = numpy.rollaxis(data,2)
435 # data = numpy.rollaxis(data,2)
437 #
436 #
438 # arrayData[:,i,:] = data
437 # arrayData[:,i,:] = data
439
438
440 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
439 arrayData = numpy.zeros((nRecords,nChannels,nPoints,nSamples))
441 doSqueeze = False
440 doSqueeze = False
442 if mode == 0:
441 if mode == 0:
443 strds = 'channel'
442 strds = 'channel'
444 nDatas = nChannels
443 nDatas = nChannels
445 newShapes = (nRecords,nPoints,nSamples)
444 newShapes = (nRecords,nPoints,nSamples)
446 if nPoints == 1:
445 if nPoints == 1:
447 doSqueeze = True
446 doSqueeze = True
448 axisSqueeze = 2
447 axisSqueeze = 2
449 else:
448 else:
450 strds = 'param'
449 strds = 'param'
451 nDatas = nPoints
450 nDatas = nPoints
452 newShapes = (nRecords,nChannels,nSamples)
451 newShapes = (nRecords,nChannels,nSamples)
453 if nChannels == 1:
452 if nChannels == 1:
454 doSqueeze = True
453 doSqueeze = True
455 axisSqueeze = 1
454 axisSqueeze = 1
456
455
457 for i in range(nDatas):
456 for i in range(nDatas):
458
457
459 data = dataset[strds + str(i)].value
458 data = dataset[strds + str(i)].value
460 data = data.reshape(newShapes)
459 data = data.reshape(newShapes)
461
460
462 if mode == 0:
461 if mode == 0:
463 arrayData[:,i,:,:] = data
462 arrayData[:,i,:,:] = data
464 else:
463 else:
465 arrayData[:,:,i,:] = data
464 arrayData[:,:,i,:] = data
466
465
467 if doSqueeze:
466 if doSqueeze:
468 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
467 arrayData = numpy.squeeze(arrayData, axis=axisSqueeze)
469
468
470 return arrayData
469 return arrayData
471
470
472 def __initDataOut(self, type):
471 def __initDataOut(self, type):
473
472
474 # if type =='Parameters':
473 # if type =='Parameters':
475 # self.dataOut = Parameters()
474 # self.dataOut = Parameters()
476 # elif type =='Spectra':
475 # elif type =='Spectra':
477 # self.dataOut = Spectra()
476 # self.dataOut = Spectra()
478 # elif type =='Voltage':
477 # elif type =='Voltage':
479 # self.dataOut = Voltage()
478 # self.dataOut = Voltage()
480 # elif type =='Correlation':
479 # elif type =='Correlation':
481 # self.dataOut = Correlation()
480 # self.dataOut = Correlation()
482
481
483 return
482 return
484
483
485 def __setDataOut(self):
484 def __setDataOut(self):
486 listMeta = self.listMeta
485 listMeta = self.listMeta
487 listMetaname = self.listMetaname
486 listMetaname = self.listMetaname
488 listDataname = self.listDataname
487 listDataname = self.listDataname
489 listData = self.listData
488 listData = self.listData
490
489
491 blockIndex = self.blockIndex
490 blockIndex = self.blockIndex
492 blockList = self.blockList
491 blockList = self.blockList
493
492
494 for i in range(len(listMeta)):
493 for i in range(len(listMeta)):
495 setattr(self.dataOut,listMetaname[i],listMeta[i])
494 setattr(self.dataOut,listMetaname[i],listMeta[i])
496
495
497 for j in range(len(listData)):
496 for j in range(len(listData)):
498 if listDataname[j]=='utctime':
497 if listDataname[j]=='utctime':
499 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
498 # setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex]])
500 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
499 setattr(self.dataOut,'utctimeInit',listData[j][blockList[blockIndex]])
501 continue
500 continue
502
501
503 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
502 setattr(self.dataOut,listDataname[j],listData[j][blockList[blockIndex],:])
504
503
505 return self.dataOut.data_param
504 return self.dataOut.data_param
506
505
507 def getData(self):
506 def getData(self):
508
507
509 # if self.flagNoMoreFiles:
508 # if self.flagNoMoreFiles:
510 # self.dataOut.flagNoData = True
509 # self.dataOut.flagNoData = True
511 # print 'Process finished'
510 # print 'Process finished'
512 # return 0
511 # return 0
513 #
512 #
514 if self.blockIndex==self.blocksPerFile:
513 if self.blockIndex==self.blocksPerFile:
515 if not( self.__setNextFileOffline() ):
514 if not( self.__setNextFileOffline() ):
516 self.dataOut.flagNoData = True
515 self.dataOut.flagNoData = True
517 return 0
516 return 0
518
517
519 #
518 #
520 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
519 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
521 # self.dataOut.flagNoData = True
520 # self.dataOut.flagNoData = True
522 # return 0
521 # return 0
523
522
524 self.__readData()
523 self.__readData()
525 self.__setDataOut()
524 self.__setDataOut()
526 self.dataOut.flagNoData = False
525 self.dataOut.flagNoData = False
527
526
528 self.blockIndex += 1
527 self.blockIndex += 1
529
528
530 return
529 return
531
530
532 def run(self, **kwargs):
531 def run(self, **kwargs):
533
532
534 if not(self.isConfig):
533 if not(self.isConfig):
535 self.setup(**kwargs)
534 self.setup(**kwargs)
536 # self.setObjProperties()
535 # self.setObjProperties()
537 self.isConfig = True
536 self.isConfig = True
538
537
539 self.getData()
538 self.getData()
540
539
541 return
540 return
542
541
543 class HDF5Writer(Operation):
542 class HDF5Writer(Operation):
544
543
545 ext = ".hdf5"
544 ext = ".hdf5"
546
545
547 optchar = "D"
546 optchar = "D"
548
547
549 metaoptchar = "M"
548 metaoptchar = "M"
550
549
551 metaFile = None
550 metaFile = None
552
551
553 filename = None
552 filename = None
554
553
555 path = None
554 path = None
556
555
557 setFile = None
556 setFile = None
558
557
559 fp = None
558 fp = None
560
559
561 grp = None
560 grp = None
562
561
563 ds = None
562 ds = None
564
563
565 firsttime = True
564 firsttime = True
566
565
567 #Configurations
566 #Configurations
568
567
569 blocksPerFile = None
568 blocksPerFile = None
570
569
571 blockIndex = None
570 blockIndex = None
572
571
573 dataOut = None
572 dataOut = None
574
573
575 #Data Arrays
574 #Data Arrays
576
575
577 dataList = None
576 dataList = None
578
577
579 metadataList = None
578 metadataList = None
580
579
581 arrayDim = None
580 arrayDim = None
582
581
583 tableDim = None
582 tableDim = None
584
583
585 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
584 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
586
585
587 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
586 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
588
587
589 mode = None
588 mode = None
590
589
591 nDatas = None #Number of datasets to be stored per array
590 nDatas = None #Number of datasets to be stored per array
592
591
593 nDims = None #Number Dimensions in each dataset
592 nDims = None #Number Dimensions in each dataset
594
593
595 nDimsForDs = None
594 nDimsForDs = None
596
595
597 def __init__(self):
596 def __init__(self):
598
597
599 Operation.__init__(self)
598 Operation.__init__(self)
600 self.isConfig = False
599 self.isConfig = False
601 return
600 return
602
601
603
602
604 def setup(self, dataOut, **kwargs):
603 def setup(self, dataOut, **kwargs):
605
604
606 self.path = kwargs['path']
605 self.path = kwargs['path']
607
606
608 if kwargs.has_key('ext'):
607 if kwargs.has_key('ext'):
609 self.ext = kwargs['ext']
608 self.ext = kwargs['ext']
610
609
611 if kwargs.has_key('blocksPerFile'):
610 if kwargs.has_key('blocksPerFile'):
612 self.blocksPerFile = kwargs['blocksPerFile']
611 self.blocksPerFile = kwargs['blocksPerFile']
613 else:
612 else:
614 self.blocksPerFile = 10
613 self.blocksPerFile = 10
615
614
616 self.metadataList = kwargs['metadataList']
615 self.metadataList = kwargs['metadataList']
617
616
618 self.dataList = kwargs['dataList']
617 self.dataList = kwargs['dataList']
619
618
620 self.dataOut = dataOut
619 self.dataOut = dataOut
621
620
622 if kwargs.has_key('mode'):
621 if kwargs.has_key('mode'):
623 mode = kwargs['mode']
622 mode = kwargs['mode']
624
623
625 if type(mode) == int:
624 if type(mode) == int:
626 mode = numpy.zeros(len(self.dataList)) + mode
625 mode = numpy.zeros(len(self.dataList)) + mode
627 else:
626 else:
628 mode = numpy.zeros(len(self.dataList))
627 mode = numpy.zeros(len(self.dataList))
629
628
630 self.mode = mode
629 self.mode = mode
631
630
632 arrayDim = numpy.zeros((len(self.dataList),5))
631 arrayDim = numpy.zeros((len(self.dataList),5))
633
632
634 #Table dimensions
633 #Table dimensions
635
634
636 dtype0 = self.dtype
635 dtype0 = self.dtype
637
636
638 tableList = []
637 tableList = []
639
638
640 for i in range(len(self.dataList)):
639 for i in range(len(self.dataList)):
641
640
642 dataAux = getattr(self.dataOut, self.dataList[i])
641 dataAux = getattr(self.dataOut, self.dataList[i])
643
642
644 if type(dataAux)==float or type(dataAux)==int:
643 if type(dataAux)==float or type(dataAux)==int:
645 arrayDim[i,0] = 1
644 arrayDim[i,0] = 1
646 else:
645 else:
647 arrayDim0 = dataAux.shape
646 arrayDim0 = dataAux.shape
648 arrayDim[i,0] = len(arrayDim0)
647 arrayDim[i,0] = len(arrayDim0)
649 arrayDim[i,4] = mode[i]
648 arrayDim[i,4] = mode[i]
650
649
651 if len(arrayDim0) == 3:
650 if len(arrayDim0) == 3:
652 arrayDim[i,1:-1] = numpy.array(arrayDim0)
651 arrayDim[i,1:-1] = numpy.array(arrayDim0)
653 elif len(arrayDim0) == 2:
652 elif len(arrayDim0) == 2:
654 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
653 arrayDim[i,2:-1] = numpy.array(arrayDim0) #nHeights
655 elif len(arrayDim0) == 1:
654 elif len(arrayDim0) == 1:
656 arrayDim[i,3] = arrayDim0
655 arrayDim[i,3] = arrayDim0
657 elif len(arrayDim0) == 0:
656 elif len(arrayDim0) == 0:
658 arrayDim[i,0] = 1
657 arrayDim[i,0] = 1
659 arrayDim[i,3] = 1
658 arrayDim[i,3] = 1
660
659
661 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
660 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
662 tableList.append(table)
661 tableList.append(table)
663
662
664 self.arrayDim = arrayDim
663 self.arrayDim = arrayDim
665 self.tableDim = numpy.array(tableList, dtype = dtype0)
664 self.tableDim = numpy.array(tableList, dtype = dtype0)
666 self.blockIndex = 0
665 self.blockIndex = 0
667
666
668 return
667 return
669
668
670 def putMetadata(self):
669 def putMetadata(self):
671
670
672 fp = self.createMetadataFile()
671 fp = self.createMetadataFile()
673 self.writeMetadata(fp)
672 self.writeMetadata(fp)
674 fp.close()
673 fp.close()
675 return
674 return
676
675
677 def createMetadataFile(self):
676 def createMetadataFile(self):
678 ext = self.ext
677 ext = self.ext
679 path = self.path
678 path = self.path
680 setFile = self.setFile
679 setFile = self.setFile
681
680
682 timeTuple = time.localtime(self.dataOut.utctime)
681 timeTuple = time.localtime(self.dataOut.utctime)
683
682
684 subfolder = ''
683 subfolder = ''
685 fullpath = os.path.join( path, subfolder )
684 fullpath = os.path.join( path, subfolder )
686
685
687 if not( os.path.exists(fullpath) ):
686 if not( os.path.exists(fullpath) ):
688 os.mkdir(fullpath)
687 os.mkdir(fullpath)
689 setFile = -1 #inicializo mi contador de seteo
688 setFile = -1 #inicializo mi contador de seteo
690
689
691 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
690 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
692 fullpath = os.path.join( path, subfolder )
691 fullpath = os.path.join( path, subfolder )
693
692
694 if not( os.path.exists(fullpath) ):
693 if not( os.path.exists(fullpath) ):
695 os.mkdir(fullpath)
694 os.mkdir(fullpath)
696 setFile = -1 #inicializo mi contador de seteo
695 setFile = -1 #inicializo mi contador de seteo
697
696
698 else:
697 else:
699 filesList = os.listdir( fullpath )
698 filesList = os.listdir( fullpath )
700 filesList = sorted( filesList, key=str.lower )
699 filesList = sorted( filesList, key=str.lower )
701 if len( filesList ) > 0:
700 if len( filesList ) > 0:
702 filesList = [k for k in filesList if 'M' in k]
701 filesList = [k for k in filesList if 'M' in k]
703 filen = filesList[-1]
702 filen = filesList[-1]
704 # el filename debera tener el siguiente formato
703 # el filename debera tener el siguiente formato
705 # 0 1234 567 89A BCDE (hex)
704 # 0 1234 567 89A BCDE (hex)
706 # x YYYY DDD SSS .ext
705 # x YYYY DDD SSS .ext
707 if isNumber( filen[8:11] ):
706 if isNumber( filen[8:11] ):
708 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
707 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
709 else:
708 else:
710 setFile = -1
709 setFile = -1
711 else:
710 else:
712 setFile = -1 #inicializo mi contador de seteo
711 setFile = -1 #inicializo mi contador de seteo
713
712
714 setFile += 1
713 setFile += 1
715
714
716 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
715 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
717 timeTuple.tm_year,
716 timeTuple.tm_year,
718 timeTuple.tm_yday,
717 timeTuple.tm_yday,
719 setFile,
718 setFile,
720 ext )
719 ext )
721
720
722 filename = os.path.join( path, subfolder, file )
721 filename = os.path.join( path, subfolder, file )
723 self.metaFile = file
722 self.metaFile = file
724 #Setting HDF5 File
723 #Setting HDF5 File
725 fp = h5py.File(filename,'w')
724 fp = h5py.File(filename,'w')
726
725
727 return fp
726 return fp
728
727
729 def writeMetadata(self, fp):
728 def writeMetadata(self, fp):
730
729
731 grp = fp.create_group("Metadata")
730 grp = fp.create_group("Metadata")
732 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
731 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
733
732
734 for i in range(len(self.metadataList)):
733 for i in range(len(self.metadataList)):
735 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
734 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
736 return
735 return
737
736
738 def setNextFile(self):
737 def setNextFile(self):
739
738
740 ext = self.ext
739 ext = self.ext
741 path = self.path
740 path = self.path
742 setFile = self.setFile
741 setFile = self.setFile
743 mode = self.mode
742 mode = self.mode
744
743
745 timeTuple = time.localtime(self.dataOut.utctime)
744 timeTuple = time.localtime(self.dataOut.utctime)
746 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
745 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
747
746
748 fullpath = os.path.join( path, subfolder )
747 fullpath = os.path.join( path, subfolder )
749
748
750 if os.path.exists(fullpath):
749 if os.path.exists(fullpath):
751 filesList = os.listdir( fullpath )
750 filesList = os.listdir( fullpath )
752 filesList = [k for k in filesList if 'D' in k]
751 filesList = [k for k in filesList if 'D' in k]
753 if len( filesList ) > 0:
752 if len( filesList ) > 0:
754 filesList = sorted( filesList, key=str.lower )
753 filesList = sorted( filesList, key=str.lower )
755 filen = filesList[-1]
754 filen = filesList[-1]
756 # el filename debera tener el siguiente formato
755 # el filename debera tener el siguiente formato
757 # 0 1234 567 89A BCDE (hex)
756 # 0 1234 567 89A BCDE (hex)
758 # x YYYY DDD SSS .ext
757 # x YYYY DDD SSS .ext
759 if isNumber( filen[8:11] ):
758 if isNumber( filen[8:11] ):
760 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
759 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
761 else:
760 else:
762 setFile = -1
761 setFile = -1
763 else:
762 else:
764 setFile = -1 #inicializo mi contador de seteo
763 setFile = -1 #inicializo mi contador de seteo
765
764
766 setFile += 1
765 setFile += 1
767
766
768 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
767 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
769 timeTuple.tm_year,
768 timeTuple.tm_year,
770 timeTuple.tm_yday,
769 timeTuple.tm_yday,
771 setFile,
770 setFile,
772 ext )
771 ext )
773
772
774 filename = os.path.join( path, subfolder, file )
773 filename = os.path.join( path, subfolder, file )
775
774
776 #Setting HDF5 File
775 #Setting HDF5 File
777 fp = h5py.File(filename,'w')
776 fp = h5py.File(filename,'w')
778 grp = fp.create_group("Data")
777 grp = fp.create_group("Data")
779 grp.attrs['metadata'] = self.metaFile
778 grp.attrs['metadata'] = self.metaFile
780
779
781 # grp.attrs['blocksPerFile'] = 0
780 # grp.attrs['blocksPerFile'] = 0
782
781
783 ds = []
782 ds = []
784 data = []
783 data = []
785 nDimsForDs = []
784 nDimsForDs = []
786
785
787 nDatas = numpy.zeros(len(self.dataList))
786 nDatas = numpy.zeros(len(self.dataList))
788 nDims = self.arrayDim[:,0]
787 nDims = self.arrayDim[:,0]
789
788
790 nDim1 = self.arrayDim[:,2]
789 nDim1 = self.arrayDim[:,2]
791 nDim0 = self.arrayDim[:,3]
790 nDim0 = self.arrayDim[:,3]
792
791
793 for i in range(len(self.dataList)):
792 for i in range(len(self.dataList)):
794
793
795 if nDims[i]==1:
794 if nDims[i]==1:
796 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
795 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
797 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
796 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
798 ds.append(ds0)
797 ds.append(ds0)
799 data.append([])
798 data.append([])
800 nDimsForDs.append(nDims[i])
799 nDimsForDs.append(nDims[i])
801 else:
800 else:
802
801
803 if mode[i]==0:
802 if mode[i]==0:
804 strMode = "channel"
803 strMode = "channel"
805 nDatas[i] = self.arrayDim[i,1]
804 nDatas[i] = self.arrayDim[i,1]
806 else:
805 else:
807 strMode = "param"
806 strMode = "param"
808 nDatas[i] = self.arrayDim[i,2]
807 nDatas[i] = self.arrayDim[i,2]
809
808
810 if nDims[i]==2:
809 if nDims[i]==2:
811 nDatas[i] = self.arrayDim[i,2]
810 nDatas[i] = self.arrayDim[i,2]
812
811
813 grp0 = grp.create_group(self.dataList[i])
812 grp0 = grp.create_group(self.dataList[i])
814
813
815 for j in range(int(nDatas[i])):
814 for j in range(int(nDatas[i])):
816 tableName = strMode + str(j)
815 tableName = strMode + str(j)
817
816
818 if nDims[i] == 3:
817 if nDims[i] == 3:
819 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
818 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
820 else:
819 else:
821 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
820 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
822
821
823 ds.append(ds0)
822 ds.append(ds0)
824 data.append([])
823 data.append([])
825 nDimsForDs.append(nDims[i])
824 nDimsForDs.append(nDims[i])
826 self.nDatas = nDatas
825 self.nDatas = nDatas
827 self.nDims = nDims
826 self.nDims = nDims
828 self.nDimsForDs = nDimsForDs
827 self.nDimsForDs = nDimsForDs
829 #Saving variables
828 #Saving variables
830 print 'Writing the file: %s'%filename
829 print 'Writing the file: %s'%filename
831 self.filename = filename
830 self.filename = filename
832 self.fp = fp
831 self.fp = fp
833 self.grp = grp
832 self.grp = grp
834 self.grp.attrs.modify('nRecords', 1)
833 self.grp.attrs.modify('nRecords', 1)
835 self.ds = ds
834 self.ds = ds
836 self.data = data
835 self.data = data
837
836
838 self.setFile = setFile
837 self.setFile = setFile
839 self.firsttime = True
838 self.firsttime = True
840 self.blockIndex = 0
839 self.blockIndex = 0
841 return
840 return
842
841
843 def putData(self):
842 def putData(self):
844
843
845 if not self.firsttime:
844 if not self.firsttime:
846 self.readBlock()
845 self.readBlock()
847
846
848 if self.blockIndex == self.blocksPerFile:
847 if self.blockIndex == self.blocksPerFile:
849
848
850 self.setNextFile()
849 self.setNextFile()
851
850
852 self.setBlock()
851 self.setBlock()
853 self.writeBlock()
852 self.writeBlock()
854
853
855 self.fp.flush()
854 self.fp.flush()
856 self.fp.close()
855 self.fp.close()
857
856
858 return
857 return
859
858
860 def readBlock(self):
859 def readBlock(self):
861
860
862 '''
861 '''
863 data Array configured
862 data Array configured
864
863
865
864
866 self.data
865 self.data
867 '''
866 '''
868 ds = self.ds
867 ds = self.ds
869 #Setting HDF5 File
868 #Setting HDF5 File
870 fp = h5py.File(self.filename,'r+')
869 fp = h5py.File(self.filename,'r+')
871 grp = fp["Data"]
870 grp = fp["Data"]
872 ind = 0
871 ind = 0
873
872
874 # grp.attrs['blocksPerFile'] = 0
873 # grp.attrs['blocksPerFile'] = 0
875 for i in range(len(self.dataList)):
874 for i in range(len(self.dataList)):
876
875
877 if self.nDims[i]==1:
876 if self.nDims[i]==1:
878 ds0 = grp[self.dataList[i]]
877 ds0 = grp[self.dataList[i]]
879 ds[ind] = ds0
878 ds[ind] = ds0
880 ind += 1
879 ind += 1
881 else:
880 else:
882 if self.mode[i]==0:
881 if self.mode[i]==0:
883 strMode = "channel"
882 strMode = "channel"
884 else:
883 else:
885 strMode = "param"
884 strMode = "param"
886
885
887 grp0 = grp[self.dataList[i]]
886 grp0 = grp[self.dataList[i]]
888
887
889 for j in range(int(self.nDatas[i])):
888 for j in range(int(self.nDatas[i])):
890 tableName = strMode + str(j)
889 tableName = strMode + str(j)
891 ds0 = grp0[tableName]
890 ds0 = grp0[tableName]
892 ds[ind] = ds0
891 ds[ind] = ds0
893 ind += 1
892 ind += 1
894
893
895
894
896 self.fp = fp
895 self.fp = fp
897 self.grp = grp
896 self.grp = grp
898 self.ds = ds
897 self.ds = ds
899
898
900 return
899 return
901
900
902
901
903 def setBlock(self):
902 def setBlock(self):
904 '''
903 '''
905 data Array configured
904 data Array configured
906
905
907
906
908 self.data
907 self.data
909 '''
908 '''
910 #Creating Arrays
909 #Creating Arrays
911 data = self.data
910 data = self.data
912 nDatas = self.nDatas
911 nDatas = self.nDatas
913 nDims = self.nDims
912 nDims = self.nDims
914 mode = self.mode
913 mode = self.mode
915 ind = 0
914 ind = 0
916
915
917 for i in range(len(self.dataList)):
916 for i in range(len(self.dataList)):
918 dataAux = getattr(self.dataOut,self.dataList[i])
917 dataAux = getattr(self.dataOut,self.dataList[i])
919
918
920 if nDims[i] == 1:
919 if nDims[i] == 1:
921 # data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
920 # data[ind] = numpy.array([str(dataAux)]).reshape((1,1))
922 data[ind] = dataAux
921 data[ind] = dataAux
923 # if not self.firsttime:
922 # if not self.firsttime:
924 # data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
923 # data[ind] = numpy.hstack((self.ds[ind][:], self.data[ind]))
925 ind += 1
924 ind += 1
926 else:
925 else:
927 for j in range(int(nDatas[i])):
926 for j in range(int(nDatas[i])):
928 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
927 if (mode[i] == 0) or (nDims[i] == 2): #In case division per channel or Dimensions is only 1
929 data[ind] = dataAux[j,:]
928 data[ind] = dataAux[j,:]
930 else:
929 else:
931 data[ind] = dataAux[:,j,:]
930 data[ind] = dataAux[:,j,:]
932
931
933 # if nDims[i] == 3:
932 # if nDims[i] == 3:
934 # data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
933 # data[ind] = data[ind].reshape((data[ind].shape[0],data[ind].shape[1],1))
935
934
936 # if not self.firsttime:
935 # if not self.firsttime:
937 # data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
936 # data[ind] = numpy.dstack((self.ds[ind][:], data[ind]))
938
937
939 # else:
938 # else:
940 # data[ind] = data[ind].reshape((1,data[ind].shape[0]))
939 # data[ind] = data[ind].reshape((1,data[ind].shape[0]))
941
940
942 # if not self.firsttime:
941 # if not self.firsttime:
943 # data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
942 # data[ind] = numpy.vstack((self.ds[ind][:], data[ind]))
944 ind += 1
943 ind += 1
945
944
946 self.data = data
945 self.data = data
947 return
946 return
948
947
949 def writeBlock(self):
948 def writeBlock(self):
950 '''
949 '''
951 Saves the block in the HDF5 file
950 Saves the block in the HDF5 file
952 '''
951 '''
953 for i in range(len(self.ds)):
952 for i in range(len(self.ds)):
954 if self.firsttime:
953 if self.firsttime:
955 # self.ds[i].resize(self.data[i].shape)
954 # self.ds[i].resize(self.data[i].shape)
956 # self.ds[i][self.blockIndex,:] = self.data[i]
955 # self.ds[i][self.blockIndex,:] = self.data[i]
957 if type(self.data[i]) == numpy.ndarray:
956 if type(self.data[i]) == numpy.ndarray:
958 nDims1 = len(self.ds[i].shape)
957 nDims1 = len(self.ds[i].shape)
959
958
960 if nDims1 == 3:
959 if nDims1 == 3:
961 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
960 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
962
961
963 self.ds[i].resize(self.data[i].shape)
962 self.ds[i].resize(self.data[i].shape)
964 self.ds[i][:] = self.data[i]
963 self.ds[i][:] = self.data[i]
965 else:
964 else:
966 if self.nDimsForDs[i] == 1:
965 if self.nDimsForDs[i] == 1:
967 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
966 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
968 self.ds[i][0,-1] = self.data[i]
967 self.ds[i][0,-1] = self.data[i]
969 elif self.nDimsForDs[i] == 2:
968 elif self.nDimsForDs[i] == 2:
970 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
969 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
971 self.ds[i][self.blockIndex,:] = self.data[i]
970 self.ds[i][self.blockIndex,:] = self.data[i]
972 elif self.nDimsForDs[i] == 3:
971 elif self.nDimsForDs[i] == 3:
973
972
974 dataShape = self.data[i].shape
973 dataShape = self.data[i].shape
975 dsShape = self.ds[i].shape
974 dsShape = self.ds[i].shape
976
975
977 if dataShape[0]==dsShape[0]:
976 if dataShape[0]==dsShape[0]:
978 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
977 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
979 self.ds[i][:,:,-1] = self.data[i]
978 self.ds[i][:,:,-1] = self.data[i]
980 else:
979 else:
981 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2]))
980 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1],self.ds[i].shape[2]))
982 self.ds[i][dsShape[0]:,:,0] = self.data[i]
981 self.ds[i][dsShape[0]:,:,0] = self.data[i]
983 # self.ds[i].append(self.data[i])
982 # self.ds[i].append(self.data[i])
984 # self.fp.flush()
983 # self.fp.flush()
985 # if not self.firsttime:
984 # if not self.firsttime:
986 # self.fp.root.Data._v_attrs.nRecords = self.blockIndex
985 # self.fp.root.Data._v_attrs.nRecords = self.blockIndex
987
986
988 # if self.firsttime:
987 # if self.firsttime:
989 # self.fp.close()
988 # self.fp.close()
990 # self.readBlock2()
989 # self.readBlock2()
991
990
992 self.blockIndex += 1
991 self.blockIndex += 1
993 self.firsttime = False
992 self.firsttime = False
994 return
993 return
995
994
996 def run(self, dataOut, **kwargs):
995 def run(self, dataOut, **kwargs):
997 if not(self.isConfig):
996 if not(self.isConfig):
998 self.setup(dataOut, **kwargs)
997 self.setup(dataOut, **kwargs)
999 self.isConfig = True
998 self.isConfig = True
1000 self.putMetadata()
999 self.putMetadata()
1001 self.setNextFile()
1000 self.setNextFile()
1002
1001
1003 self.putData()
1002 self.putData()
1004 return
1003 return
1005
1004
General Comments 0
You need to be logged in to leave comments. Login now