##// END OF EJS Templates
schain-jc
RSS
-bug fix in HDF5 Writer Module
Julio Valdez -
r810:7e14525bff70
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1054 +1,1201
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class HDF5Reader(ProcessingUnit):
15 class HDF5Reader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self):
77 def __init__(self):
78 self.dataOut = Parameters()
78 self.dataOut = Parameters()
79 return
79 return
80
80
81 def setup(self, **kwargs):
81 def setup(self, **kwargs):
82
82
83 path = kwargs['path']
83 path = kwargs['path']
84 startDate = kwargs['startDate']
84 startDate = kwargs['startDate']
85 endDate = kwargs['endDate']
85 endDate = kwargs['endDate']
86 startTime = kwargs['startTime']
86 startTime = kwargs['startTime']
87 endTime = kwargs['endTime']
87 endTime = kwargs['endTime']
88 walk = kwargs['walk']
88 walk = kwargs['walk']
89 if kwargs.has_key('ext'):
89 if kwargs.has_key('ext'):
90 ext = kwargs['ext']
90 ext = kwargs['ext']
91 else:
91 else:
92 ext = '.hdf5'
92 ext = '.hdf5'
93
93
94 print "[Reading] Searching files in offline mode ..."
94 print "[Reading] Searching files in offline mode ..."
95 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
95 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
96 startTime=startTime, endTime=endTime,
96 startTime=startTime, endTime=endTime,
97 ext=ext, walk=walk)
97 ext=ext, walk=walk)
98
98
99 if not(filenameList):
99 if not(filenameList):
100 print "There is no files into the folder: %s"%(path)
100 print "There is no files into the folder: %s"%(path)
101 sys.exit(-1)
101 sys.exit(-1)
102
102
103 self.fileIndex = -1
103 self.fileIndex = -1
104 self.startTime = startTime
104 self.startTime = startTime
105 self.endTime = endTime
105 self.endTime = endTime
106
106
107 self.__readMetadata()
107 self.__readMetadata()
108
108
109 self.__setNextFileOffline()
109 self.__setNextFileOffline()
110
110
111 return
111 return
112
112
113 def __searchFilesOffLine(self,
113 def __searchFilesOffLine(self,
114 path,
114 path,
115 startDate=None,
115 startDate=None,
116 endDate=None,
116 endDate=None,
117 startTime=datetime.time(0,0,0),
117 startTime=datetime.time(0,0,0),
118 endTime=datetime.time(23,59,59),
118 endTime=datetime.time(23,59,59),
119 ext='.hdf5',
119 ext='.hdf5',
120 walk=True):
120 walk=True):
121
121
122 expLabel = ''
122 expLabel = ''
123 self.filenameList = []
123 self.filenameList = []
124 self.datetimeList = []
124 self.datetimeList = []
125
125
126 pathList = []
126 pathList = []
127
127
128 JRODataObj = JRODataReader()
128 JRODataObj = JRODataReader()
129 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
129 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
130
130
131 if dateList == []:
131 if dateList == []:
132 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
132 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
133 datetime.datetime.combine(startDate,startTime).ctime(),
133 datetime.datetime.combine(startDate,startTime).ctime(),
134 datetime.datetime.combine(endDate,endTime).ctime())
134 datetime.datetime.combine(endDate,endTime).ctime())
135
135
136 return None, None
136 return None, None
137
137
138 if len(dateList) > 1:
138 if len(dateList) > 1:
139 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
139 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
140 else:
140 else:
141 print "[Reading] data was found for the date %s" %(dateList[0])
141 print "[Reading] data was found for the date %s" %(dateList[0])
142
142
143 filenameList = []
143 filenameList = []
144 datetimeList = []
144 datetimeList = []
145
145
146 #----------------------------------------------------------------------------------
146 #----------------------------------------------------------------------------------
147
147
148 for thisPath in pathList:
148 for thisPath in pathList:
149 # thisPath = pathList[pathDict[file]]
149 # thisPath = pathList[pathDict[file]]
150
150
151 fileList = glob.glob1(thisPath, "*%s" %ext)
151 fileList = glob.glob1(thisPath, "*%s" %ext)
152 fileList.sort()
152 fileList.sort()
153
153
154 for file in fileList:
154 for file in fileList:
155
155
156 filename = os.path.join(thisPath,file)
156 filename = os.path.join(thisPath,file)
157
157
158 if not isFileInDateRange(filename, startDate, endDate):
158 if not isFileInDateRange(filename, startDate, endDate):
159 continue
159 continue
160
160
161 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
161 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
162
162
163 if not(thisDatetime):
163 if not(thisDatetime):
164 continue
164 continue
165
165
166 filenameList.append(filename)
166 filenameList.append(filename)
167 datetimeList.append(thisDatetime)
167 datetimeList.append(thisDatetime)
168
168
169 if not(filenameList):
169 if not(filenameList):
170 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
170 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
171 return None, None
171 return None, None
172
172
173 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
173 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
174 print
174 print
175
175
176 for i in range(len(filenameList)):
176 for i in range(len(filenameList)):
177 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
177 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
178
178
179 self.filenameList = filenameList
179 self.filenameList = filenameList
180 self.datetimeList = datetimeList
180 self.datetimeList = datetimeList
181
181
182 return pathList, filenameList
182 return pathList, filenameList
183
183
184 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
184 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
185
185
186 """
186 """
187 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
187 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
188
188
189 Inputs:
189 Inputs:
190 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
190 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
191
191
192 startDate : fecha inicial del rango seleccionado en formato datetime.date
192 startDate : fecha inicial del rango seleccionado en formato datetime.date
193
193
194 endDate : fecha final del rango seleccionado en formato datetime.date
194 endDate : fecha final del rango seleccionado en formato datetime.date
195
195
196 startTime : tiempo inicial del rango seleccionado en formato datetime.time
196 startTime : tiempo inicial del rango seleccionado en formato datetime.time
197
197
198 endTime : tiempo final del rango seleccionado en formato datetime.time
198 endTime : tiempo final del rango seleccionado en formato datetime.time
199
199
200 Return:
200 Return:
201 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
201 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
202 fecha especificado, de lo contrario retorna False.
202 fecha especificado, de lo contrario retorna False.
203
203
204 Excepciones:
204 Excepciones:
205 Si el archivo no existe o no puede ser abierto
205 Si el archivo no existe o no puede ser abierto
206 Si la cabecera no puede ser leida.
206 Si la cabecera no puede ser leida.
207
207
208 """
208 """
209
209
210 try:
210 try:
211 fp = h5py.File(filename,'r')
211 fp = h5py.File(filename,'r')
212 grp1 = fp['Data']
212 grp1 = fp['Data']
213
213
214 except IOError:
214 except IOError:
215 traceback.print_exc()
215 traceback.print_exc()
216 raise IOError, "The file %s can't be opened" %(filename)
216 raise IOError, "The file %s can't be opened" %(filename)
217 #chino rata
217 #chino rata
218 #In case has utctime attribute
218 #In case has utctime attribute
219 grp2 = grp1['utctime']
219 grp2 = grp1['utctime']
220 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
220 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
221 thisUtcTime = grp2.value[0]
221 thisUtcTime = grp2.value[0]
222
222
223 fp.close()
223 fp.close()
224
224
225 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
225 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
226 thisDate = thisDatetime.date()
226 thisDate = thisDatetime.date()
227 thisTime = thisDatetime.time()
227 thisTime = thisDatetime.time()
228
228
229 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
229 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
230 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
230 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
231
231
232 #General case
232 #General case
233 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
233 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
234 #-----------o----------------------------o-----------
234 #-----------o----------------------------o-----------
235 # startTime endTime
235 # startTime endTime
236
236
237 if endTime >= startTime:
237 if endTime >= startTime:
238 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
238 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
239 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
239 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
240 return thisDatetime
240 return thisDatetime
241 return None
241 return None
242
242
243 #If endTime < startTime then endTime belongs to the next day
243 #If endTime < startTime then endTime belongs to the next day
244 #<<<<<<<<<<<o o>>>>>>>>>>>
244 #<<<<<<<<<<<o o>>>>>>>>>>>
245 #-----------o----------------------------o-----------
245 #-----------o----------------------------o-----------
246 # endTime startTime
246 # endTime startTime
247
247
248 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
248 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
249 return None
249 return None
250
250
251 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
251 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
252 return None
252 return None
253
253
254 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
254 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
255 return None
255 return None
256
256
257 return thisDatetime
257 return thisDatetime
258
258
259 def __setNextFileOffline(self):
259 def __setNextFileOffline(self):
260
260
261 self.fileIndex += 1
261 self.fileIndex += 1
262 idFile = self.fileIndex
262 idFile = self.fileIndex
263
263
264 if not(idFile < len(self.filenameList)):
264 if not(idFile < len(self.filenameList)):
265 print "No more Files"
265 print "No more Files"
266 return 0
266 return 0
267
267
268 filename = self.filenameList[idFile]
268 filename = self.filenameList[idFile]
269
269
270 filePointer = h5py.File(filename,'r')
270 filePointer = h5py.File(filename,'r')
271
271
272 self.filename = filename
272 self.filename = filename
273
273
274 self.fp = filePointer
274 self.fp = filePointer
275
275
276 print "Setting the file: %s"%self.filename
276 print "Setting the file: %s"%self.filename
277
277
278 # self.__readMetadata()
278 # self.__readMetadata()
279 self.__setBlockList()
279 self.__setBlockList()
280 self.__readData()
280 self.__readData()
281 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
281 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
282 # self.nRecords = self.fp['Data'].attrs['nRecords']
282 # self.nRecords = self.fp['Data'].attrs['nRecords']
283 self.blockIndex = 0
283 self.blockIndex = 0
284 return 1
284 return 1
285
285
286 def __setBlockList(self):
286 def __setBlockList(self):
287 '''
287 '''
288 Selects the data within the times defined
288 Selects the data within the times defined
289
289
290 self.fp
290 self.fp
291 self.startTime
291 self.startTime
292 self.endTime
292 self.endTime
293
293
294 self.blockList
294 self.blockList
295 self.blocksPerFile
295 self.blocksPerFile
296
296
297 '''
297 '''
298 fp = self.fp
298 fp = self.fp
299 startTime = self.startTime
299 startTime = self.startTime
300 endTime = self.endTime
300 endTime = self.endTime
301
301
302 grp = fp['Data']
302 grp = fp['Data']
303 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
303 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
304
304
305 if self.timezone == 'lt':
305 if self.timezone == 'lt':
306 thisUtcTime -= 5*3600
306 thisUtcTime -= 5*3600
307
307
308 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
308 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
309 thisDate = thisDatetime.date()
309 thisDate = thisDatetime.date()
310 thisTime = thisDatetime.time()
310 thisTime = thisDatetime.time()
311
311
312 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
312 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
313 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
313 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
314
314
315 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
315 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
316
316
317 self.blockList = ind
317 self.blockList = ind
318 self.blocksPerFile = len(ind)
318 self.blocksPerFile = len(ind)
319
319
320 return
320 return
321
321
322 def __readMetadata(self):
322 def __readMetadata(self):
323 '''
323 '''
324 Reads Metadata
324 Reads Metadata
325
325
326 self.pathMeta
326 self.pathMeta
327
327
328 self.listShapes
328 self.listShapes
329 self.listMetaname
329 self.listMetaname
330 self.listMeta
330 self.listMeta
331
331
332 '''
332 '''
333
333
334 # grp = self.fp['Data']
334 # grp = self.fp['Data']
335 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
335 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
336 #
336 #
337 # if pathMeta == self.pathMeta:
337 # if pathMeta == self.pathMeta:
338 # return
338 # return
339 # else:
339 # else:
340 # self.pathMeta = pathMeta
340 # self.pathMeta = pathMeta
341 #
341 #
342 # filePointer = h5py.File(self.pathMeta,'r')
342 # filePointer = h5py.File(self.pathMeta,'r')
343 # groupPointer = filePointer['Metadata']
343 # groupPointer = filePointer['Metadata']
344
344
345 filename = self.filenameList[0]
345 filename = self.filenameList[0]
346
346
347 fp = h5py.File(filename,'r')
347 fp = h5py.File(filename,'r')
348
348
349 gp = fp['Metadata']
349 gp = fp['Metadata']
350
350
351 listMetaname = []
351 listMetaname = []
352 listMetadata = []
352 listMetadata = []
353 for item in gp.items():
353 for item in gp.items():
354 name = item[0]
354 name = item[0]
355
355
356 if name=='array dimensions':
356 if name=='array dimensions':
357 table = gp[name][:]
357 table = gp[name][:]
358 listShapes = {}
358 listShapes = {}
359 for shapes in table:
359 for shapes in table:
360 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
360 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
361 else:
361 else:
362 data = gp[name].value
362 data = gp[name].value
363 listMetaname.append(name)
363 listMetaname.append(name)
364 listMetadata.append(data)
364 listMetadata.append(data)
365
365
366 # if name=='type':
366 # if name=='type':
367 # self.__initDataOut(data)
367 # self.__initDataOut(data)
368
368
369 self.listShapes = listShapes
369 self.listShapes = listShapes
370 self.listMetaname = listMetaname
370 self.listMetaname = listMetaname
371 self.listMeta = listMetadata
371 self.listMeta = listMetadata
372
372
373 fp.close()
373 fp.close()
374 return
374 return
375
375
376 def __readData(self):
376 def __readData(self):
377 grp = self.fp['Data']
377 grp = self.fp['Data']
378 listdataname = []
378 listdataname = []
379 listdata = []
379 listdata = []
380
380
381 for item in grp.items():
381 for item in grp.items():
382 name = item[0]
382 name = item[0]
383 listdataname.append(name)
383 listdataname.append(name)
384
384
385 array = self.__setDataArray(grp[name],self.listShapes[name])
385 array = self.__setDataArray(grp[name],self.listShapes[name])
386 listdata.append(array)
386 listdata.append(array)
387
387
388 self.listDataname = listdataname
388 self.listDataname = listdataname
389 self.listData = listdata
389 self.listData = listdata
390 return
390 return
391
391
392 def __setDataArray(self, dataset, shapes):
392 def __setDataArray(self, dataset, shapes):
393
393
394 nDims = shapes[0]
394 nDims = shapes[0]
395
395
396 nDim2 = shapes[1] #Dimension 0
396 nDim2 = shapes[1] #Dimension 0
397
397
398 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
398 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
399
399
400 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
400 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
401
401
402 mode = shapes[4] #Mode of storing
402 mode = shapes[4] #Mode of storing
403
403
404 blockList = self.blockList
404 blockList = self.blockList
405
405
406 blocksPerFile = self.blocksPerFile
406 blocksPerFile = self.blocksPerFile
407
407
408 #Depending on what mode the data was stored
408 #Depending on what mode the data was stored
409 # if mode == 0: #Divided in channels
409 # if mode == 0: #Divided in channels
410 # strds = 'channel'
410 # strds = 'channel'
411 # nDatas = nDim2
411 # nDatas = nDim2
412 # newShapes = (blocksPerFile,nDim1,nDim0)
412 # newShapes = (blocksPerFile,nDim1,nDim0)
413 if mode == 1: #Divided in parameter
413 if mode == 1: #Divided in parameter
414 strds = 'param'
414 strds = 'param'
415 nDatas = nDim1
415 nDatas = nDim1
416 newShapes = (blocksPerFile,nDim2,nDim0)
416 newShapes = (blocksPerFile,nDim2,nDim0)
417 elif mode==2: #Concatenated in a table
417 elif mode==2: #Concatenated in a table
418 strds = 'table0'
418 strds = 'table0'
419 arrayData = dataset[strds].value
419 arrayData = dataset[strds].value
420 #Selecting part of the dataset
420 #Selecting part of the dataset
421 utctime = arrayData[:,0]
421 utctime = arrayData[:,0]
422 u, indices = numpy.unique(utctime, return_index=True)
422 u, indices = numpy.unique(utctime, return_index=True)
423
423
424 if blockList.size != indices.size:
424 if blockList.size != indices.size:
425 indMin = indices[blockList[0]]
425 indMin = indices[blockList[0]]
426 indMax = indices[blockList[-1] + 1]
426 indMax = indices[blockList[-1] + 1]
427 arrayData = arrayData[indMin:indMax,:]
427 arrayData = arrayData[indMin:indMax,:]
428 return arrayData
428 return arrayData
429
429
430 #------- One dimension ---------------
430 #------- One dimension ---------------
431 if nDims == 1:
431 if nDims == 1:
432 arrayData = dataset.value.astype(numpy.float)[0][blockList]
432 arrayData = dataset.value.astype(numpy.float)[0][blockList]
433
433
434 #------- Two dimensions -----------
434 #------- Two dimensions -----------
435 elif nDims == 2:
435 elif nDims == 2:
436 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
436 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
437 newShapes = (blocksPerFile,nDim0)
437 newShapes = (blocksPerFile,nDim0)
438 nDatas = nDim1
438 nDatas = nDim1
439
439
440 for i in range(nDatas):
440 for i in range(nDatas):
441 data = dataset[strds + str(i)].value
441 data = dataset[strds + str(i)].value
442 arrayData[:,i,:] = data[blockList,:]
442 arrayData[:,i,:] = data[blockList,:]
443
443
444 #------- Three dimensions ---------
444 #------- Three dimensions ---------
445 else:
445 else:
446 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
446 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
447 for i in range(nDatas):
447 for i in range(nDatas):
448
448
449 data = dataset[strds + str(i)].value
449 data = dataset[strds + str(i)].value
450 data = data[blockList,:,:]
450 data = data[blockList,:,:]
451 data = data.reshape(newShapes)
451 data = data.reshape(newShapes)
452 # if mode == 0:
452 # if mode == 0:
453 # arrayData[:,i,:,:] = data
453 # arrayData[:,i,:,:] = data
454 # else:
454 # else:
455 arrayData[:,:,i,:] = data
455 arrayData[:,:,i,:] = data
456
456
457 return arrayData
457 return arrayData
458
458
459 def __setDataOut(self):
459 def __setDataOut(self):
460 listMeta = self.listMeta
460 listMeta = self.listMeta
461 listMetaname = self.listMetaname
461 listMetaname = self.listMetaname
462 listDataname = self.listDataname
462 listDataname = self.listDataname
463 listData = self.listData
463 listData = self.listData
464 listShapes = self.listShapes
464 listShapes = self.listShapes
465
465
466 blockIndex = self.blockIndex
466 blockIndex = self.blockIndex
467 # blockList = self.blockList
467 # blockList = self.blockList
468
468
469 for i in range(len(listMeta)):
469 for i in range(len(listMeta)):
470 setattr(self.dataOut,listMetaname[i],listMeta[i])
470 setattr(self.dataOut,listMetaname[i],listMeta[i])
471
471
472 for j in range(len(listData)):
472 for j in range(len(listData)):
473 nShapes = listShapes[listDataname[j]][0]
473 nShapes = listShapes[listDataname[j]][0]
474 mode = listShapes[listDataname[j]][4]
474 mode = listShapes[listDataname[j]][4]
475 if nShapes == 1:
475 if nShapes == 1:
476 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
476 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
477 elif nShapes > 1:
477 elif nShapes > 1:
478 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
478 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
479 #Mode Meteors
479 #Mode Meteors
480 elif mode ==2:
480 elif mode ==2:
481 selectedData = self.__selectDataMode2(listData[j], blockIndex)
481 selectedData = self.__selectDataMode2(listData[j], blockIndex)
482 setattr(self.dataOut, listDataname[j], selectedData)
482 setattr(self.dataOut, listDataname[j], selectedData)
483 return
483 return
484
484
485 def __selectDataMode2(self, data, blockIndex):
485 def __selectDataMode2(self, data, blockIndex):
486 utctime = data[:,0]
486 utctime = data[:,0]
487 aux, indices = numpy.unique(utctime, return_inverse=True)
487 aux, indices = numpy.unique(utctime, return_inverse=True)
488 selInd = numpy.where(indices == blockIndex)[0]
488 selInd = numpy.where(indices == blockIndex)[0]
489 selData = data[selInd,:]
489 selData = data[selInd,:]
490
490
491 return selData
491 return selData
492
492
493 def getData(self):
493 def getData(self):
494
494
495 # if self.flagNoMoreFiles:
495 # if self.flagNoMoreFiles:
496 # self.dataOut.flagNoData = True
496 # self.dataOut.flagNoData = True
497 # print 'Process finished'
497 # print 'Process finished'
498 # return 0
498 # return 0
499 #
499 #
500 if self.blockIndex==self.blocksPerFile:
500 if self.blockIndex==self.blocksPerFile:
501 if not( self.__setNextFileOffline() ):
501 if not( self.__setNextFileOffline() ):
502 self.dataOut.flagNoData = True
502 self.dataOut.flagNoData = True
503 return 0
503 return 0
504
504
505 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
505 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
506 # self.dataOut.flagNoData = True
506 # self.dataOut.flagNoData = True
507 # return 0
507 # return 0
508 # self.__readData()
508 # self.__readData()
509 self.__setDataOut()
509 self.__setDataOut()
510 self.dataOut.flagNoData = False
510 self.dataOut.flagNoData = False
511
511
512 self.blockIndex += 1
512 self.blockIndex += 1
513
513
514 return
514 return
515
515
516 def run(self, **kwargs):
516 def run(self, **kwargs):
517
517
518 if not(self.isConfig):
518 if not(self.isConfig):
519 self.setup(**kwargs)
519 self.setup(**kwargs)
520 # self.setObjProperties()
520 # self.setObjProperties()
521 self.isConfig = True
521 self.isConfig = True
522
522
523 self.getData()
523 self.getData()
524
524
525 return
525 return
526
526
527 class HDF5Writer(Operation):
527 class HDF5Writer(Operation):
528 '''
528 '''
529 HDF5 Writer, stores parameters data in HDF5 format files
529 HDF5 Writer, stores parameters data in HDF5 format files
530
530
531 path: path where the files will be stored
531 path: path where the files will be stored
532
532
533 blocksPerFile: number of blocks that will be saved in per HDF5 format file
533 blocksPerFile: number of blocks that will be saved in per HDF5 format file
534
534
535 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
535 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
536
536
537 metadataList: list of attributes that will be stored as metadata
537 metadataList: list of attributes that will be stored as metadata
538
538
539 dataList: list of attributes that will be stores as data
539 dataList: list of attributes that will be stores as data
540
540
541 '''
541 '''
542
542
543
543
544 ext = ".hdf5"
544 ext = ".hdf5"
545
545
546 optchar = "D"
546 optchar = "D"
547
547
548 metaoptchar = "M"
548 metaoptchar = "M"
549
549
550 metaFile = None
550 metaFile = None
551
551
552 filename = None
552 filename = None
553
553
554 path = None
554 path = None
555
555
556 setFile = None
556 setFile = None
557
557
558 fp = None
558 fp = None
559
559
560 grp = None
560 grp = None
561
561
562 ds = None
562 ds = None
563
563
564 firsttime = True
564 firsttime = True
565
565
566 #Configurations
566 #Configurations
567
567
568 blocksPerFile = None
568 blocksPerFile = None
569
569
570 blockIndex = None
570 blockIndex = None
571
571
572 dataOut = None
572 dataOut = None
573
573
574 #Data Arrays
574 #Data Arrays
575
575
576 dataList = None
576 dataList = None
577
577
578 metadataList = None
578 metadataList = None
579
579
580 arrayDim = None
580 # arrayDim = None
581
582 dsList = None #List of dictionaries
581
583
582 tableDim = None
584 tableDim = None
583
585
584 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
586 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
585
587
586 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
588 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
587
589
588 mode = None
590 mode = None
589
591
590 nDatas = None #Number of datasets to be stored per array
592 nDatas = None #Number of datasets to be stored per array
591
593
592 nDims = None #Number Dimensions in each dataset
594 nDims = None #Number Dimensions in each dataset
593
595
594 nDimsForDs = None
596 nDimsForDs = None
595
597
596 currentDay = None
598 currentDay = None
597
599
598 def __init__(self):
600 def __init__(self):
599
601
600 Operation.__init__(self)
602 Operation.__init__(self)
601 self.isConfig = False
603 self.isConfig = False
602 return
604 return
603
605
604 def setup(self, dataOut, **kwargs):
606 def setup(self, dataOut, **kwargs):
605
607
606 self.path = kwargs['path']
608 self.path = kwargs['path']
607
609
608 if kwargs.has_key('blocksPerFile'):
610 if kwargs.has_key('blocksPerFile'):
609 self.blocksPerFile = kwargs['blocksPerFile']
611 self.blocksPerFile = kwargs['blocksPerFile']
610 else:
612 else:
611 self.blocksPerFile = 10
613 self.blocksPerFile = 10
612
614
613 self.metadataList = kwargs['metadataList']
615 self.metadataList = kwargs['metadataList']
614
615 self.dataList = kwargs['dataList']
616 self.dataList = kwargs['dataList']
616
617 self.dataOut = dataOut
617 self.dataOut = dataOut
618
618
619 if kwargs.has_key('mode'):
619 if kwargs.has_key('mode'):
620 mode = kwargs['mode']
620 mode = kwargs['mode']
621
621
622 if type(mode) == int:
622 if type(mode) == int:
623 mode = numpy.zeros(len(self.dataList)) + mode
623 mode = numpy.zeros(len(self.dataList)) + mode
624 else:
624 else:
625 mode = numpy.ones(len(self.dataList))
625 mode = numpy.ones(len(self.dataList))
626
626
627 self.mode = mode
627 self.mode = mode
628
628
629 arrayDim = numpy.zeros((len(self.dataList),5))
629 arrayDim = numpy.zeros((len(self.dataList),5))
630
630
631 #Table dimensions
631 #Table dimensions
632
633 dtype0 = self.dtype
632 dtype0 = self.dtype
634
635 tableList = []
633 tableList = []
636
634
635 #Dictionary and list of tables
636 dsList = []
637
637 for i in range(len(self.dataList)):
638 for i in range(len(self.dataList)):
638
639 dsDict = {}
639 dataAux = getattr(self.dataOut, self.dataList[i])
640 dataAux = getattr(self.dataOut, self.dataList[i])
640
641 dsDict['variable'] = self.dataList[i]
641 #--------------------- Conditionals ------------------------
642 #--------------------- Conditionals ------------------------
642 #There is no data
643 #There is no data
643 if dataAux == None:
644 if dataAux == None:
644 return 0
645 return 0
645
646
646 #Not array, just a number
647 #Not array, just a number
647 if type(dataAux)==float or type(dataAux)==int:
648 #Mode 0
649 if type(dataAux)==float or type(dataAux)==int:
650 dsDict['mode'] = 0
651 dsDict['nDim'] = 0
648 arrayDim[i,0] = 1
652 arrayDim[i,0] = 1
649 mode[i] = 0
653 dsList.append(dsDict)
650
654
651 #Mode meteors
655 #Mode 2: meteors
652 elif mode[i] == 2:
656 elif mode[i] == 2:
657 # dsDict['nDim'] = 0
658 dsDict['dsName'] = 'table0'
659 dsDict['mode'] = 2 # Mode meteors
660 dsDict['shape'] = dataAux.shape[-1]
661 dsDict['nDim'] = 0
662
653 arrayDim[i,3] = dataAux.shape[-1]
663 arrayDim[i,3] = dataAux.shape[-1]
654 arrayDim[i,4] = mode[i] #Mode the data was stored
664 arrayDim[i,4] = mode[i] #Mode the data was stored
665
666 dsList.append(dsDict)
655
667
656 #All the rest
668 #Mode 1
657 else:
669 else:
658 arrayDim0 = dataAux.shape #Data dimensions
670 arrayDim0 = dataAux.shape #Data dimensions
659 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
671 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
660 arrayDim[i,4] = mode[i] #Mode the data was stored
672 arrayDim[i,4] = mode[i] #Mode the data was stored
661
673
674 strtable = 'table'
675 dsDict['mode'] = 1 # Mode parameters
676
662 # Three-dimension arrays
677 # Three-dimension arrays
663 if len(arrayDim0) == 3:
678 if len(arrayDim0) == 3:
664 arrayDim[i,1:-1] = numpy.array(arrayDim0)
679 arrayDim[i,1:-1] = numpy.array(arrayDim0)
665
680 nTables = int(arrayDim[i,2])
681 dsDict['dsNumber'] = nTables
682 dsDict['shape'] = arrayDim[i,2:4]
683 dsDict['nDim'] = 3
684
685 for j in range(nTables):
686 dsDict = dsDict.copy()
687 dsDict['dsName'] = strtable + str(j)
688 dsList.append(dsDict)
689
666 # Two-dimension arrays
690 # Two-dimension arrays
667 elif len(arrayDim0) == 2:
691 elif len(arrayDim0) == 2:
668 arrayDim[i,2:-1] = numpy.array(arrayDim0)
692 arrayDim[i,2:-1] = numpy.array(arrayDim0)
669
693 nTables = int(arrayDim[i,2])
694 dsDict['dsNumber'] = nTables
695 dsDict['shape'] = arrayDim[i,3]
696 dsDict['nDim'] = 2
697
698 for j in range(nTables):
699 dsDict = dsDict.copy()
700 dsDict['dsName'] = strtable + str(j)
701 dsList.append(dsDict)
702
670 # One-dimension arrays
703 # One-dimension arrays
671 elif len(arrayDim0) == 1:
704 elif len(arrayDim0) == 1:
672 arrayDim[i,3] = arrayDim0
705 arrayDim[i,3] = arrayDim0[0]
673
706 dsDict['shape'] = arrayDim0[0]
674 # No array, just a number
707 dsDict['dsNumber'] = 1
675 elif len(arrayDim0) == 0:
708 dsDict['dsName'] = strtable + str(0)
676 arrayDim[i,0] = 1
709 dsDict['nDim'] = 1
677 arrayDim[i,3] = 1
710 dsList.append(dsDict)
678
711
679 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
712 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
680 tableList.append(table)
713 tableList.append(table)
681
714
682 self.arrayDim = arrayDim
715 # self.arrayDim = arrayDim
716 self.dsList = dsList
683 self.tableDim = numpy.array(tableList, dtype = dtype0)
717 self.tableDim = numpy.array(tableList, dtype = dtype0)
684 self.blockIndex = 0
718 self.blockIndex = 0
685
719
686 timeTuple = time.localtime(dataOut.utctime)
720 timeTuple = time.localtime(dataOut.utctime)
687 self.currentDay = timeTuple.tm_yday
721 self.currentDay = timeTuple.tm_yday
688 return 1
722 return 1
689
723
690 def putMetadata(self):
724 def putMetadata(self):
691
725
692 fp = self.createMetadataFile()
726 fp = self.createMetadataFile()
693 self.writeMetadata(fp)
727 self.writeMetadata(fp)
694 fp.close()
728 fp.close()
695 return
729 return
696
730
697 def createMetadataFile(self):
731 def createMetadataFile(self):
698 ext = self.ext
732 ext = self.ext
699 path = self.path
733 path = self.path
700 setFile = self.setFile
734 setFile = self.setFile
701
735
702 timeTuple = time.localtime(self.dataOut.utctime)
736 timeTuple = time.localtime(self.dataOut.utctime)
703
737
704 subfolder = ''
738 subfolder = ''
705 fullpath = os.path.join( path, subfolder )
739 fullpath = os.path.join( path, subfolder )
706
740
707 if not( os.path.exists(fullpath) ):
741 if not( os.path.exists(fullpath) ):
708 os.mkdir(fullpath)
742 os.mkdir(fullpath)
709 setFile = -1 #inicializo mi contador de seteo
743 setFile = -1 #inicializo mi contador de seteo
710
744
711 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
745 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
712 fullpath = os.path.join( path, subfolder )
746 fullpath = os.path.join( path, subfolder )
713
747
714 if not( os.path.exists(fullpath) ):
748 if not( os.path.exists(fullpath) ):
715 os.mkdir(fullpath)
749 os.mkdir(fullpath)
716 setFile = -1 #inicializo mi contador de seteo
750 setFile = -1 #inicializo mi contador de seteo
717
751
718 else:
752 else:
719 filesList = os.listdir( fullpath )
753 filesList = os.listdir( fullpath )
720 filesList = sorted( filesList, key=str.lower )
754 filesList = sorted( filesList, key=str.lower )
721 if len( filesList ) > 0:
755 if len( filesList ) > 0:
722 filesList = [k for k in filesList if 'M' in k]
756 filesList = [k for k in filesList if 'M' in k]
723 filen = filesList[-1]
757 filen = filesList[-1]
724 # el filename debera tener el siguiente formato
758 # el filename debera tener el siguiente formato
725 # 0 1234 567 89A BCDE (hex)
759 # 0 1234 567 89A BCDE (hex)
726 # x YYYY DDD SSS .ext
760 # x YYYY DDD SSS .ext
727 if isNumber( filen[8:11] ):
761 if isNumber( filen[8:11] ):
728 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
762 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
729 else:
763 else:
730 setFile = -1
764 setFile = -1
731 else:
765 else:
732 setFile = -1 #inicializo mi contador de seteo
766 setFile = -1 #inicializo mi contador de seteo
733
767
734 setFile += 1
768 setFile += 1
735
769
736 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
770 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
737 timeTuple.tm_year,
771 timeTuple.tm_year,
738 timeTuple.tm_yday,
772 timeTuple.tm_yday,
739 setFile,
773 setFile,
740 ext )
774 ext )
741
775
742 filename = os.path.join( path, subfolder, file )
776 filename = os.path.join( path, subfolder, file )
743 self.metaFile = file
777 self.metaFile = file
744 #Setting HDF5 File
778 #Setting HDF5 File
745 fp = h5py.File(filename,'w')
779 fp = h5py.File(filename,'w')
746
780
747 return fp
781 return fp
748
782
749 def writeMetadata(self, fp):
783 def writeMetadata(self, fp):
750
784
751 grp = fp.create_group("Metadata")
785 grp = fp.create_group("Metadata")
752 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
786 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
753
787
754 for i in range(len(self.metadataList)):
788 for i in range(len(self.metadataList)):
755 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
789 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
756 return
790 return
757
791
758 def dateFlag(self):
792 def dateFlag(self):
759
793
760 timeTuple = time.localtime(self.dataOut.utctime)
794 timeTuple = time.localtime(self.dataOut.utctime)
761 dataDay = timeTuple.tm_yday
795 dataDay = timeTuple.tm_yday
762
796
763 if dataDay == self.currentDay:
797 if dataDay == self.currentDay:
764 return False
798 return False
765
799
766 self.currentDay = dataDay
800 self.currentDay = dataDay
767 return True
801 return True
768
802
769 def setNextFile(self):
803 def setNextFile(self):
770
804
771 ext = self.ext
805 ext = self.ext
772 path = self.path
806 path = self.path
773 setFile = self.setFile
807 setFile = self.setFile
774 mode = self.mode
808 mode = self.mode
775
809
776 timeTuple = time.localtime(self.dataOut.utctime)
810 timeTuple = time.localtime(self.dataOut.utctime)
777 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
811 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
778
812
779 fullpath = os.path.join( path, subfolder )
813 fullpath = os.path.join( path, subfolder )
780
814
781 if os.path.exists(fullpath):
815 if os.path.exists(fullpath):
782 filesList = os.listdir( fullpath )
816 filesList = os.listdir( fullpath )
783 filesList = [k for k in filesList if 'D' in k]
817 filesList = [k for k in filesList if 'D' in k]
784 if len( filesList ) > 0:
818 if len( filesList ) > 0:
785 filesList = sorted( filesList, key=str.lower )
819 filesList = sorted( filesList, key=str.lower )
786 filen = filesList[-1]
820 filen = filesList[-1]
787 # el filename debera tener el siguiente formato
821 # el filename debera tener el siguiente formato
788 # 0 1234 567 89A BCDE (hex)
822 # 0 1234 567 89A BCDE (hex)
789 # x YYYY DDD SSS .ext
823 # x YYYY DDD SSS .ext
790 if isNumber( filen[8:11] ):
824 if isNumber( filen[8:11] ):
791 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
825 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
792 else:
826 else:
793 setFile = -1
827 setFile = -1
794 else:
828 else:
795 setFile = -1 #inicializo mi contador de seteo
829 setFile = -1 #inicializo mi contador de seteo
796 else:
830 else:
797 os.mkdir(fullpath)
831 os.mkdir(fullpath)
798 setFile = -1 #inicializo mi contador de seteo
832 setFile = -1 #inicializo mi contador de seteo
799
833
800 setFile += 1
834 setFile += 1
801
835
802 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
836 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
803 timeTuple.tm_year,
837 timeTuple.tm_year,
804 timeTuple.tm_yday,
838 timeTuple.tm_yday,
805 setFile,
839 setFile,
806 ext )
840 ext )
807
841
808 filename = os.path.join( path, subfolder, file )
842 filename = os.path.join( path, subfolder, file )
809
843
810 #Setting HDF5 File
844 #Setting HDF5 File
811 fp = h5py.File(filename,'w')
845 fp = h5py.File(filename,'w')
846 #write metadata
847 self.writeMetadata(fp)
848 #Write data
849 grp = fp.create_group("Data")
850 # grp.attrs['metadata'] = self.metaFile
851
852 # grp.attrs['blocksPerFile'] = 0
853 ds = []
854 data = []
855 dsList = self.dsList
856 i = 0
857 while i < len(dsList):
858 dsInfo = dsList[i]
859 #One-dimension data
860 if dsInfo['mode'] == 0:
861 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
862 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
863 ds.append(ds0)
864 data.append([])
865 i += 1
866 continue
867 # nDimsForDs.append(nDims[i])
868
869 elif dsInfo['mode'] == 2:
870 grp0 = grp.create_group(dsInfo['variable'])
871 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
872 ds.append(ds0)
873 data.append([])
874 i += 1
875 continue
876
877 elif dsInfo['mode'] == 1:
878 grp0 = grp.create_group(dsInfo['variable'])
879
880 for j in range(dsInfo['dsNumber']):
881 dsInfo = dsList[i]
882 tableName = dsInfo['dsName']
883 shape = dsInfo['shape']
884
885 if dsInfo['nDim'] == 3:
886 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
887 else:
888 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
889
890 ds.append(ds0)
891 data.append([])
892 i += 1
893 # nDimsForDs.append(nDims[i])
894
895 fp.flush()
896 fp.close()
897
898 # self.nDatas = nDatas
899 # self.nDims = nDims
900 # self.nDimsForDs = nDimsForDs
901 #Saving variables
902 print 'Writing the file: %s'%filename
903 self.filename = filename
904 # self.fp = fp
905 # self.grp = grp
906 # self.grp.attrs.modify('nRecords', 1)
907 self.ds = ds
908 self.data = data
909 # self.setFile = setFile
910 self.firsttime = True
911 self.blockIndex = 0
912 return
913
914 def setNextFile1(self):
915
916 ext = self.ext
917 path = self.path
918 setFile = self.setFile
919 mode = self.mode
920
921 timeTuple = time.localtime(self.dataOut.utctime)
922 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
923
924 fullpath = os.path.join( path, subfolder )
925
926 if os.path.exists(fullpath):
927 filesList = os.listdir( fullpath )
928 filesList = [k for k in filesList if 'D' in k]
929 if len( filesList ) > 0:
930 filesList = sorted( filesList, key=str.lower )
931 filen = filesList[-1]
932 # el filename debera tener el siguiente formato
933 # 0 1234 567 89A BCDE (hex)
934 # x YYYY DDD SSS .ext
935 if isNumber( filen[8:11] ):
936 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
937 else:
938 setFile = -1
939 else:
940 setFile = -1 #inicializo mi contador de seteo
941 else:
942 os.mkdir(fullpath)
943 setFile = -1 #inicializo mi contador de seteo
944
945 setFile += 1
946
947 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
948 timeTuple.tm_year,
949 timeTuple.tm_yday,
950 setFile,
951 ext )
952
953 filename = os.path.join( path, subfolder, file )
954
955 #Setting HDF5 File
956 fp = h5py.File(filename,'w')
812
957
813 #writemetadata
958 #writemetadata
814 self.writeMetadata(fp)
959 self.writeMetadata(fp)
815
960
816 grp = fp.create_group("Data")
961 grp = fp.create_group("Data")
817 # grp.attrs['metadata'] = self.metaFile
962 # grp.attrs['metadata'] = self.metaFile
818
963
819 # grp.attrs['blocksPerFile'] = 0
964 # grp.attrs['blocksPerFile'] = 0
820
965
821 ds = []
966 ds = []
822 data = []
967 data = []
823 nDimsForDs = []
968 nDimsForDs = []
824
969
825 nDatas = numpy.zeros(len(self.dataList))
970 nDatas = numpy.zeros(len(self.dataList))
826 nDims = self.arrayDim[:,0]
971 nDims = self.arrayDim[:,0]
827
972
828 nDim1 = self.arrayDim[:,2]
973 nDim1 = self.arrayDim[:,2]
829 nDim0 = self.arrayDim[:,3]
974 nDim0 = self.arrayDim[:,3]
830
975
831 for i in range(len(self.dataList)):
976 for i in range(len(self.dataList)):
832
977
833 #One-dimension data
978 #One-dimension data
834 if nDims[i]==1:
979 if nDims[i]==1:
835 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
980 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
836 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
981 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
837 ds.append(ds0)
982 ds.append(ds0)
838 data.append([])
983 data.append([])
839 nDimsForDs.append(nDims[i])
984 nDimsForDs.append(nDims[i])
840 else:
985 else:
841
986
842 #Channel mode
987 #Channel mode
843 # if mode[i] == 0:
988 # if mode[i] == 0:
844 # strMode = "channel"
989 # strMode = "channel"
845 #
990 #
846 # #nDatas is the number of arrays per variable
991 # #nDatas is the number of arrays per variable
847 # if nDims[i] == 1:
992 # if nDims[i] == 1:
848 # nDatas[i] = self.arrayDim[i,1]
993 # nDatas[i] = self.arrayDim[i,1]
849 # elif nDims[i] == 2:
994 # elif nDims[i] == 2:
850 # nDatas[i] = self.arrayDim[i,2]
995 # nDatas[i] = self.arrayDim[i,2]
851
996
852 #Parameters mode
997 #Parameters mode
853 if mode[i] == 1:
998 if mode[i] == 1:
854 strMode = "param"
999 strMode = "param"
855 nDatas[i] = self.arrayDim[i,2]
1000 nDatas[i] = self.arrayDim[i,2]
856
1001
857 #Meteors mode
1002 #Meteors mode
858 elif mode[i] == 2:
1003 elif mode[i] == 2:
859 strMode = "table"
1004 strMode = "table"
860 nDatas[i] = 1
1005 nDatas[i] = 1
861
1006
862 grp0 = grp.create_group(self.dataList[i])
1007 grp0 = grp.create_group(self.dataList[i])
863
1008
864 for j in range(int(nDatas[i])):
1009 for j in range(int(nDatas[i])):
865 tableName = strMode + str(j)
1010 tableName = strMode + str(j)
866
1011
867 if nDims[i] == 3:
1012 if nDims[i] == 3:
868 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
1013 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
869
1014
870 else:
1015 else:
871 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
1016 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
872
1017
873 ds.append(ds0)
1018 ds.append(ds0)
874 data.append([])
1019 data.append([])
875 nDimsForDs.append(nDims[i])
1020 nDimsForDs.append(nDims[i])
876
1021
877 fp.flush()
1022 fp.flush()
878 fp.close()
1023 fp.close()
879
1024
880 self.nDatas = nDatas
1025 self.nDatas = nDatas
881 self.nDims = nDims
1026 self.nDims = nDims
882 self.nDimsForDs = nDimsForDs
1027 self.nDimsForDs = nDimsForDs
883 #Saving variables
1028 #Saving variables
884 print 'Writing the file: %s'%filename
1029 print 'Writing the file: %s'%filename
885 self.filename = filename
1030 self.filename = filename
886 # self.fp = fp
1031 # self.fp = fp
887 # self.grp = grp
1032 # self.grp = grp
888 # self.grp.attrs.modify('nRecords', 1)
1033 # self.grp.attrs.modify('nRecords', 1)
889 self.ds = ds
1034 self.ds = ds
890 self.data = data
1035 self.data = data
891 #
1036 #
892 # self.setFile = setFile
1037 # self.setFile = setFile
893 self.firsttime = True
1038 self.firsttime = True
894 self.blockIndex = 0
1039 self.blockIndex = 0
895 return
1040 return
896
1041
897 def putData(self):
1042 def putData(self):
898
1043
899 if self.blockIndex == self.blocksPerFile or self.dateFlag():
1044 if self.blockIndex == self.blocksPerFile or self.dateFlag():
900 self.setNextFile()
1045 self.setNextFile()
901
1046
902 # if not self.firsttime:
1047 # if not self.firsttime:
903 self.readBlock()
1048 self.readBlock()
904 self.setBlock() #Prepare data to be written
1049 self.setBlock() #Prepare data to be written
905 self.writeBlock() #Write data
1050 self.writeBlock() #Write data
906
1051
907 return
1052 return
908
1053
909 def readBlock(self):
1054 def readBlock(self):
910
1055
911 '''
1056 '''
912 data Array configured
1057 data Array configured
913
1058
914
1059
915 self.data
1060 self.data
916 '''
1061 '''
1062 dsList = self.dsList
917 ds = self.ds
1063 ds = self.ds
918 #Setting HDF5 File
1064 #Setting HDF5 File
919 fp = h5py.File(self.filename,'r+')
1065 fp = h5py.File(self.filename,'r+')
920 grp = fp["Data"]
1066 grp = fp["Data"]
921 ind = 0
1067 ind = 0
922
1068
923 # grp.attrs['blocksPerFile'] = 0
1069 # grp.attrs['blocksPerFile'] = 0
924 for i in range(len(self.dataList)):
1070 while ind < len(dsList):
1071 dsInfo = dsList[ind]
925
1072
926 if self.nDims[i]==1:
1073 if dsInfo['mode'] == 0:
927 ds0 = grp[self.dataList[i]]
1074 ds0 = grp[dsInfo['variable']]
928 ds[ind] = ds0
1075 ds[ind] = ds0
929 ind += 1
1076 ind += 1
930 else:
1077 else:
931 # if self.mode[i] == 0:
932 # strMode = "channel"
933 if self.mode[i] == 1:
934 strMode = "param"
935 elif self.mode[i] == 2:
936 strMode = "table"
937
1078
938 grp0 = grp[self.dataList[i]]
1079 grp0 = grp[dsInfo['variable']]
939
1080
940 for j in range(int(self.nDatas[i])):
1081 for j in range(dsInfo['dsNumber']):
941 tableName = strMode + str(j)
1082 dsInfo = dsList[ind]
942 ds0 = grp0[tableName]
1083 ds0 = grp0[dsInfo['dsName']]
943 ds[ind] = ds0
1084 ds[ind] = ds0
944 ind += 1
1085 ind += 1
945
1086
946 self.fp = fp
1087 self.fp = fp
947 self.grp = grp
1088 self.grp = grp
948 self.ds = ds
1089 self.ds = ds
949
1090
950 return
1091 return
951
1092
952 def setBlock(self):
1093 def setBlock(self):
953 '''
1094 '''
954 data Array configured
1095 data Array configured
955
1096
956
1097
957 self.data
1098 self.data
958 '''
1099 '''
959 #Creating Arrays
1100 #Creating Arrays
1101 dsList = self.dsList
960 data = self.data
1102 data = self.data
961 nDatas = self.nDatas
962 nDims = self.nDims
963 mode = self.mode
964 ind = 0
1103 ind = 0
965
1104
966 for i in range(len(self.dataList)):
1105 while ind < len(dsList):
967 dataAux = getattr(self.dataOut,self.dataList[i])
1106 dsInfo = dsList[ind]
1107 dataAux = getattr(self.dataOut, dsInfo['variable'])
968
1108
969 if nDims[i] == 1 or mode[i] == 2:
1109 mode = dsInfo['mode']
1110 nDim = dsInfo['nDim']
1111
1112 if mode == 0 or mode == 2 or nDim == 1:
970 data[ind] = dataAux
1113 data[ind] = dataAux
971 ind += 1
1114 ind += 1
972
1115 # elif nDim == 1:
973 elif nDims[i] == 2:
1116 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
974 for j in range(int(nDatas[i])):
1117 # ind += 1
1118 elif nDim == 2:
1119 for j in range(dsInfo['dsNumber']):
975 data[ind] = dataAux[j,:]
1120 data[ind] = dataAux[j,:]
976 ind += 1
1121 ind += 1
977
1122 elif nDim == 3:
978 elif nDims[i] == 3:
1123 for j in range(dsInfo['dsNumber']):
979 for j in range(int(nDatas[i])):
980 # Extinct mode 0
981 # if (mode[i] == 0):
982 # data[ind] = dataAux[j,:,:]
983 # else:
984 data[ind] = dataAux[:,j,:]
1124 data[ind] = dataAux[:,j,:]
985 ind += 1
1125 ind += 1
986
1126
987 self.data = data
1127 self.data = data
988 return
1128 return
989
1129
990 def writeBlock(self):
1130 def writeBlock(self):
991 '''
1131 '''
992 Saves the block in the HDF5 file
1132 Saves the block in the HDF5 file
993 '''
1133 '''
1134 dsList = self.dsList
1135
994 for i in range(len(self.ds)):
1136 for i in range(len(self.ds)):
1137 dsInfo = dsList[i]
1138 nDim = dsInfo['nDim']
1139 mode = dsInfo['mode']
995
1140
996 # First time
1141 # First time
997 if self.firsttime:
1142 if self.firsttime:
998 # self.ds[i].resize(self.data[i].shape)
1143 # self.ds[i].resize(self.data[i].shape)
999 # self.ds[i][self.blockIndex,:] = self.data[i]
1144 # self.ds[i][self.blockIndex,:] = self.data[i]
1000 if type(self.data[i]) == numpy.ndarray:
1145 if type(self.data[i]) == numpy.ndarray:
1001 nDims1 = len(self.ds[i].shape)
1002
1146
1003 if nDims1 == 3:
1147 if nDim == 3:
1004 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1148 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1005
1149 self.ds[i].resize(self.data[i].shape)
1006 self.ds[i].resize(self.data[i].shape)
1007
1150
1008 self.ds[i][:] = self.data[i]
1151 self.ds[i][:] = self.data[i]
1009 else:
1152 else:
1010
1153
1011 # From second time
1154 # From second time
1012 # Meteors!
1155 # Meteors!
1013 if self.mode[i] == 2:
1156 if mode == 2:
1014 dataShape = self.data[i].shape
1157 dataShape = self.data[i].shape
1015 dsShape = self.ds[i].shape
1158 dsShape = self.ds[i].shape
1016 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1159 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1017 self.ds[i][dsShape[0]:,:] = self.data[i]
1160 self.ds[i][dsShape[0]:,:] = self.data[i]
1018 # One dimension
1161 # No dimension
1019 elif self.nDimsForDs[i] == 1:
1162 elif mode == 0:
1020 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1163 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1021 self.ds[i][0,-1] = self.data[i]
1164 self.ds[i][0,-1] = self.data[i]
1165 # One dimension
1166 elif nDim == 1:
1167 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1168 self.ds[i][-1,:] = self.data[i]
1022 # Two dimension
1169 # Two dimension
1023 elif self.nDimsForDs[i] == 2:
1170 elif nDim == 2:
1024 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1171 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1025 self.ds[i][self.blockIndex,:] = self.data[i]
1172 self.ds[i][self.blockIndex,:] = self.data[i]
1026 # Three dimensions
1173 # Three dimensions
1027 elif self.nDimsForDs[i] == 3:
1174 elif nDim == 3:
1028 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1175 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1029 self.ds[i][:,:,-1] = self.data[i]
1176 self.ds[i][:,:,-1] = self.data[i]
1030
1177
1031 self.firsttime = False
1178 self.firsttime = False
1032 self.blockIndex += 1
1179 self.blockIndex += 1
1033
1180
1034 #Close to save changes
1181 #Close to save changes
1035 self.fp.flush()
1182 self.fp.flush()
1036 self.fp.close()
1183 self.fp.close()
1037 return
1184 return
1038
1185
1039 def run(self, dataOut, **kwargs):
1186 def run(self, dataOut, **kwargs):
1040
1187
1041 if not(self.isConfig):
1188 if not(self.isConfig):
1042 flagdata = self.setup(dataOut, **kwargs)
1189 flagdata = self.setup(dataOut, **kwargs)
1043
1190
1044 if not(flagdata):
1191 if not(flagdata):
1045 return
1192 return
1046
1193
1047 self.isConfig = True
1194 self.isConfig = True
1048 # self.putMetadata()
1195 # self.putMetadata()
1049 self.setNextFile()
1196 self.setNextFile()
1050
1197
1051 self.putData()
1198 self.putData()
1052 return
1199 return
1053
1200
1054
1201
General Comments 0
You need to be logged in to leave comments. Login now