##// END OF EJS Templates
schain
RSS
Bug fixes to HDF5 Writer Unit
Julio Valdez -
r820:e3a794954261
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1201 +1,1067
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class HDF5Reader(ProcessingUnit):
15 class HDF5Reader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self):
77 def __init__(self):
78 self.dataOut = Parameters()
78 self.dataOut = Parameters()
79 return
79 return
80
80
81 def setup(self, **kwargs):
81 def setup(self, **kwargs):
82
82
83 path = kwargs['path']
83 path = kwargs['path']
84 startDate = kwargs['startDate']
84 startDate = kwargs['startDate']
85 endDate = kwargs['endDate']
85 endDate = kwargs['endDate']
86 startTime = kwargs['startTime']
86 startTime = kwargs['startTime']
87 endTime = kwargs['endTime']
87 endTime = kwargs['endTime']
88 walk = kwargs['walk']
88 walk = kwargs['walk']
89 if kwargs.has_key('ext'):
89 if kwargs.has_key('ext'):
90 ext = kwargs['ext']
90 ext = kwargs['ext']
91 else:
91 else:
92 ext = '.hdf5'
92 ext = '.hdf5'
93
93
94 print "[Reading] Searching files in offline mode ..."
94 print "[Reading] Searching files in offline mode ..."
95 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
95 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
96 startTime=startTime, endTime=endTime,
96 startTime=startTime, endTime=endTime,
97 ext=ext, walk=walk)
97 ext=ext, walk=walk)
98
98
99 if not(filenameList):
99 if not(filenameList):
100 print "There is no files into the folder: %s"%(path)
100 print "There is no files into the folder: %s"%(path)
101 sys.exit(-1)
101 sys.exit(-1)
102
102
103 self.fileIndex = -1
103 self.fileIndex = -1
104 self.startTime = startTime
104 self.startTime = startTime
105 self.endTime = endTime
105 self.endTime = endTime
106
106
107 self.__readMetadata()
107 self.__readMetadata()
108
108
109 self.__setNextFileOffline()
109 self.__setNextFileOffline()
110
110
111 return
111 return
112
112
113 def __searchFilesOffLine(self,
113 def __searchFilesOffLine(self,
114 path,
114 path,
115 startDate=None,
115 startDate=None,
116 endDate=None,
116 endDate=None,
117 startTime=datetime.time(0,0,0),
117 startTime=datetime.time(0,0,0),
118 endTime=datetime.time(23,59,59),
118 endTime=datetime.time(23,59,59),
119 ext='.hdf5',
119 ext='.hdf5',
120 walk=True):
120 walk=True):
121
121
122 expLabel = ''
122 expLabel = ''
123 self.filenameList = []
123 self.filenameList = []
124 self.datetimeList = []
124 self.datetimeList = []
125
125
126 pathList = []
126 pathList = []
127
127
128 JRODataObj = JRODataReader()
128 JRODataObj = JRODataReader()
129 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
129 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
130
130
131 if dateList == []:
131 if dateList == []:
132 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
132 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
133 datetime.datetime.combine(startDate,startTime).ctime(),
133 datetime.datetime.combine(startDate,startTime).ctime(),
134 datetime.datetime.combine(endDate,endTime).ctime())
134 datetime.datetime.combine(endDate,endTime).ctime())
135
135
136 return None, None
136 return None, None
137
137
138 if len(dateList) > 1:
138 if len(dateList) > 1:
139 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
139 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
140 else:
140 else:
141 print "[Reading] data was found for the date %s" %(dateList[0])
141 print "[Reading] data was found for the date %s" %(dateList[0])
142
142
143 filenameList = []
143 filenameList = []
144 datetimeList = []
144 datetimeList = []
145
145
146 #----------------------------------------------------------------------------------
146 #----------------------------------------------------------------------------------
147
147
148 for thisPath in pathList:
148 for thisPath in pathList:
149 # thisPath = pathList[pathDict[file]]
149 # thisPath = pathList[pathDict[file]]
150
150
151 fileList = glob.glob1(thisPath, "*%s" %ext)
151 fileList = glob.glob1(thisPath, "*%s" %ext)
152 fileList.sort()
152 fileList.sort()
153
153
154 for file in fileList:
154 for file in fileList:
155
155
156 filename = os.path.join(thisPath,file)
156 filename = os.path.join(thisPath,file)
157
157
158 if not isFileInDateRange(filename, startDate, endDate):
158 if not isFileInDateRange(filename, startDate, endDate):
159 continue
159 continue
160
160
161 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
161 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
162
162
163 if not(thisDatetime):
163 if not(thisDatetime):
164 continue
164 continue
165
165
166 filenameList.append(filename)
166 filenameList.append(filename)
167 datetimeList.append(thisDatetime)
167 datetimeList.append(thisDatetime)
168
168
169 if not(filenameList):
169 if not(filenameList):
170 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
170 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
171 return None, None
171 return None, None
172
172
173 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
173 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
174 print
174 print
175
175
176 for i in range(len(filenameList)):
176 for i in range(len(filenameList)):
177 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
177 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
178
178
179 self.filenameList = filenameList
179 self.filenameList = filenameList
180 self.datetimeList = datetimeList
180 self.datetimeList = datetimeList
181
181
182 return pathList, filenameList
182 return pathList, filenameList
183
183
184 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
184 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
185
185
186 """
186 """
187 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
187 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
188
188
189 Inputs:
189 Inputs:
190 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
190 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
191
191
192 startDate : fecha inicial del rango seleccionado en formato datetime.date
192 startDate : fecha inicial del rango seleccionado en formato datetime.date
193
193
194 endDate : fecha final del rango seleccionado en formato datetime.date
194 endDate : fecha final del rango seleccionado en formato datetime.date
195
195
196 startTime : tiempo inicial del rango seleccionado en formato datetime.time
196 startTime : tiempo inicial del rango seleccionado en formato datetime.time
197
197
198 endTime : tiempo final del rango seleccionado en formato datetime.time
198 endTime : tiempo final del rango seleccionado en formato datetime.time
199
199
200 Return:
200 Return:
201 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
201 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
202 fecha especificado, de lo contrario retorna False.
202 fecha especificado, de lo contrario retorna False.
203
203
204 Excepciones:
204 Excepciones:
205 Si el archivo no existe o no puede ser abierto
205 Si el archivo no existe o no puede ser abierto
206 Si la cabecera no puede ser leida.
206 Si la cabecera no puede ser leida.
207
207
208 """
208 """
209
209
210 try:
210 try:
211 fp = h5py.File(filename,'r')
211 fp = h5py.File(filename,'r')
212 grp1 = fp['Data']
212 grp1 = fp['Data']
213
213
214 except IOError:
214 except IOError:
215 traceback.print_exc()
215 traceback.print_exc()
216 raise IOError, "The file %s can't be opened" %(filename)
216 raise IOError, "The file %s can't be opened" %(filename)
217 #chino rata
217 #chino rata
218 #In case has utctime attribute
218 #In case has utctime attribute
219 grp2 = grp1['utctime']
219 grp2 = grp1['utctime']
220 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
220 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
221 thisUtcTime = grp2.value[0]
221 thisUtcTime = grp2.value[0]
222
222
223 fp.close()
223 fp.close()
224
224
225 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
225 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
226 thisDate = thisDatetime.date()
226 thisDate = thisDatetime.date()
227 thisTime = thisDatetime.time()
227 thisTime = thisDatetime.time()
228
228
229 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
229 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
230 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
230 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
231
231
232 #General case
232 #General case
233 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
233 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
234 #-----------o----------------------------o-----------
234 #-----------o----------------------------o-----------
235 # startTime endTime
235 # startTime endTime
236
236
237 if endTime >= startTime:
237 if endTime >= startTime:
238 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
238 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
239 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
239 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
240 return thisDatetime
240 return thisDatetime
241 return None
241 return None
242
242
243 #If endTime < startTime then endTime belongs to the next day
243 #If endTime < startTime then endTime belongs to the next day
244 #<<<<<<<<<<<o o>>>>>>>>>>>
244 #<<<<<<<<<<<o o>>>>>>>>>>>
245 #-----------o----------------------------o-----------
245 #-----------o----------------------------o-----------
246 # endTime startTime
246 # endTime startTime
247
247
248 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
248 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
249 return None
249 return None
250
250
251 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
251 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
252 return None
252 return None
253
253
254 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
254 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
255 return None
255 return None
256
256
257 return thisDatetime
257 return thisDatetime
258
258
259 def __setNextFileOffline(self):
259 def __setNextFileOffline(self):
260
260
261 self.fileIndex += 1
261 self.fileIndex += 1
262 idFile = self.fileIndex
262 idFile = self.fileIndex
263
263
264 if not(idFile < len(self.filenameList)):
264 if not(idFile < len(self.filenameList)):
265 print "No more Files"
265 print "No more Files"
266 return 0
266 return 0
267
267
268 filename = self.filenameList[idFile]
268 filename = self.filenameList[idFile]
269
269
270 filePointer = h5py.File(filename,'r')
270 filePointer = h5py.File(filename,'r')
271
271
272 self.filename = filename
272 self.filename = filename
273
273
274 self.fp = filePointer
274 self.fp = filePointer
275
275
276 print "Setting the file: %s"%self.filename
276 print "Setting the file: %s"%self.filename
277
277
278 # self.__readMetadata()
278 # self.__readMetadata()
279 self.__setBlockList()
279 self.__setBlockList()
280 self.__readData()
280 self.__readData()
281 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
281 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
282 # self.nRecords = self.fp['Data'].attrs['nRecords']
282 # self.nRecords = self.fp['Data'].attrs['nRecords']
283 self.blockIndex = 0
283 self.blockIndex = 0
284 return 1
284 return 1
285
285
286 def __setBlockList(self):
286 def __setBlockList(self):
287 '''
287 '''
288 Selects the data within the times defined
288 Selects the data within the times defined
289
289
290 self.fp
290 self.fp
291 self.startTime
291 self.startTime
292 self.endTime
292 self.endTime
293
293
294 self.blockList
294 self.blockList
295 self.blocksPerFile
295 self.blocksPerFile
296
296
297 '''
297 '''
298 fp = self.fp
298 fp = self.fp
299 startTime = self.startTime
299 startTime = self.startTime
300 endTime = self.endTime
300 endTime = self.endTime
301
301
302 grp = fp['Data']
302 grp = fp['Data']
303 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
303 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
304
304
305 if self.timezone == 'lt':
305 if self.timezone == 'lt':
306 thisUtcTime -= 5*3600
306 thisUtcTime -= 5*3600
307
307
308 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
308 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
309 thisDate = thisDatetime.date()
309 thisDate = thisDatetime.date()
310 thisTime = thisDatetime.time()
310 thisTime = thisDatetime.time()
311
311
312 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
312 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
313 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
313 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
314
314
315 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
315 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
316
316
317 self.blockList = ind
317 self.blockList = ind
318 self.blocksPerFile = len(ind)
318 self.blocksPerFile = len(ind)
319
319
320 return
320 return
321
321
322 def __readMetadata(self):
322 def __readMetadata(self):
323 '''
323 '''
324 Reads Metadata
324 Reads Metadata
325
325
326 self.pathMeta
326 self.pathMeta
327
327
328 self.listShapes
328 self.listShapes
329 self.listMetaname
329 self.listMetaname
330 self.listMeta
330 self.listMeta
331
331
332 '''
332 '''
333
333
334 # grp = self.fp['Data']
334 # grp = self.fp['Data']
335 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
335 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
336 #
336 #
337 # if pathMeta == self.pathMeta:
337 # if pathMeta == self.pathMeta:
338 # return
338 # return
339 # else:
339 # else:
340 # self.pathMeta = pathMeta
340 # self.pathMeta = pathMeta
341 #
341 #
342 # filePointer = h5py.File(self.pathMeta,'r')
342 # filePointer = h5py.File(self.pathMeta,'r')
343 # groupPointer = filePointer['Metadata']
343 # groupPointer = filePointer['Metadata']
344
344
345 filename = self.filenameList[0]
345 filename = self.filenameList[0]
346
346
347 fp = h5py.File(filename,'r')
347 fp = h5py.File(filename,'r')
348
348
349 gp = fp['Metadata']
349 gp = fp['Metadata']
350
350
351 listMetaname = []
351 listMetaname = []
352 listMetadata = []
352 listMetadata = []
353 for item in gp.items():
353 for item in gp.items():
354 name = item[0]
354 name = item[0]
355
355
356 if name=='array dimensions':
356 if name=='array dimensions':
357 table = gp[name][:]
357 table = gp[name][:]
358 listShapes = {}
358 listShapes = {}
359 for shapes in table:
359 for shapes in table:
360 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
360 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
361 else:
361 else:
362 data = gp[name].value
362 data = gp[name].value
363 listMetaname.append(name)
363 listMetaname.append(name)
364 listMetadata.append(data)
364 listMetadata.append(data)
365
365
366 # if name=='type':
366 # if name=='type':
367 # self.__initDataOut(data)
367 # self.__initDataOut(data)
368
368
369 self.listShapes = listShapes
369 self.listShapes = listShapes
370 self.listMetaname = listMetaname
370 self.listMetaname = listMetaname
371 self.listMeta = listMetadata
371 self.listMeta = listMetadata
372
372
373 fp.close()
373 fp.close()
374 return
374 return
375
375
376 def __readData(self):
376 def __readData(self):
377 grp = self.fp['Data']
377 grp = self.fp['Data']
378 listdataname = []
378 listdataname = []
379 listdata = []
379 listdata = []
380
380
381 for item in grp.items():
381 for item in grp.items():
382 name = item[0]
382 name = item[0]
383 listdataname.append(name)
383 listdataname.append(name)
384
384
385 array = self.__setDataArray(grp[name],self.listShapes[name])
385 array = self.__setDataArray(grp[name],self.listShapes[name])
386 listdata.append(array)
386 listdata.append(array)
387
387
388 self.listDataname = listdataname
388 self.listDataname = listdataname
389 self.listData = listdata
389 self.listData = listdata
390 return
390 return
391
391
392 def __setDataArray(self, dataset, shapes):
392 def __setDataArray(self, dataset, shapes):
393
393
394 nDims = shapes[0]
394 nDims = shapes[0]
395
395
396 nDim2 = shapes[1] #Dimension 0
396 nDim2 = shapes[1] #Dimension 0
397
397
398 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
398 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
399
399
400 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
400 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
401
401
402 mode = shapes[4] #Mode of storing
402 mode = shapes[4] #Mode of storing
403
403
404 blockList = self.blockList
404 blockList = self.blockList
405
405
406 blocksPerFile = self.blocksPerFile
406 blocksPerFile = self.blocksPerFile
407
407
408 #Depending on what mode the data was stored
408 #Depending on what mode the data was stored
409 # if mode == 0: #Divided in channels
409 # if mode == 0: #Divided in channels
410 # strds = 'channel'
410 # strds = 'channel'
411 # nDatas = nDim2
411 # nDatas = nDim2
412 # newShapes = (blocksPerFile,nDim1,nDim0)
412 # newShapes = (blocksPerFile,nDim1,nDim0)
413 if mode == 1: #Divided in parameter
413 if mode == 1: #Divided in parameter
414 strds = 'param'
414 strds = 'param'
415 nDatas = nDim1
415 nDatas = nDim1
416 newShapes = (blocksPerFile,nDim2,nDim0)
416 newShapes = (blocksPerFile,nDim2,nDim0)
417 elif mode==2: #Concatenated in a table
417 elif mode==2: #Concatenated in a table
418 strds = 'table0'
418 strds = 'table0'
419 arrayData = dataset[strds].value
419 arrayData = dataset[strds].value
420 #Selecting part of the dataset
420 #Selecting part of the dataset
421 utctime = arrayData[:,0]
421 utctime = arrayData[:,0]
422 u, indices = numpy.unique(utctime, return_index=True)
422 u, indices = numpy.unique(utctime, return_index=True)
423
423
424 if blockList.size != indices.size:
424 if blockList.size != indices.size:
425 indMin = indices[blockList[0]]
425 indMin = indices[blockList[0]]
426 indMax = indices[blockList[-1] + 1]
426 indMax = indices[blockList[-1] + 1]
427 arrayData = arrayData[indMin:indMax,:]
427 arrayData = arrayData[indMin:indMax,:]
428 return arrayData
428 return arrayData
429
429
430 #------- One dimension ---------------
430 #------- One dimension ---------------
431 if nDims == 1:
431 if nDims == 1:
432 arrayData = dataset.value.astype(numpy.float)[0][blockList]
432 arrayData = dataset.value.astype(numpy.float)[0][blockList]
433
433
434 #------- Two dimensions -----------
434 #------- Two dimensions -----------
435 elif nDims == 2:
435 elif nDims == 2:
436 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
436 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
437 newShapes = (blocksPerFile,nDim0)
437 newShapes = (blocksPerFile,nDim0)
438 nDatas = nDim1
438 nDatas = nDim1
439
439
440 for i in range(nDatas):
440 for i in range(nDatas):
441 data = dataset[strds + str(i)].value
441 data = dataset[strds + str(i)].value
442 arrayData[:,i,:] = data[blockList,:]
442 arrayData[:,i,:] = data[blockList,:]
443
443
444 #------- Three dimensions ---------
444 #------- Three dimensions ---------
445 else:
445 else:
446 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
446 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
447 for i in range(nDatas):
447 for i in range(nDatas):
448
448
449 data = dataset[strds + str(i)].value
449 data = dataset[strds + str(i)].value
450 data = data[blockList,:,:]
450 data = data[blockList,:,:]
451 data = data.reshape(newShapes)
451 data = data.reshape(newShapes)
452 # if mode == 0:
452 # if mode == 0:
453 # arrayData[:,i,:,:] = data
453 # arrayData[:,i,:,:] = data
454 # else:
454 # else:
455 arrayData[:,:,i,:] = data
455 arrayData[:,:,i,:] = data
456
456
457 return arrayData
457 return arrayData
458
458
459 def __setDataOut(self):
459 def __setDataOut(self):
460 listMeta = self.listMeta
460 listMeta = self.listMeta
461 listMetaname = self.listMetaname
461 listMetaname = self.listMetaname
462 listDataname = self.listDataname
462 listDataname = self.listDataname
463 listData = self.listData
463 listData = self.listData
464 listShapes = self.listShapes
464 listShapes = self.listShapes
465
465
466 blockIndex = self.blockIndex
466 blockIndex = self.blockIndex
467 # blockList = self.blockList
467 # blockList = self.blockList
468
468
469 for i in range(len(listMeta)):
469 for i in range(len(listMeta)):
470 setattr(self.dataOut,listMetaname[i],listMeta[i])
470 setattr(self.dataOut,listMetaname[i],listMeta[i])
471
471
472 for j in range(len(listData)):
472 for j in range(len(listData)):
473 nShapes = listShapes[listDataname[j]][0]
473 nShapes = listShapes[listDataname[j]][0]
474 mode = listShapes[listDataname[j]][4]
474 mode = listShapes[listDataname[j]][4]
475 if nShapes == 1:
475 if nShapes == 1:
476 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
476 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
477 elif nShapes > 1:
477 elif nShapes > 1:
478 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
478 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
479 #Mode Meteors
479 #Mode Meteors
480 elif mode ==2:
480 elif mode ==2:
481 selectedData = self.__selectDataMode2(listData[j], blockIndex)
481 selectedData = self.__selectDataMode2(listData[j], blockIndex)
482 setattr(self.dataOut, listDataname[j], selectedData)
482 setattr(self.dataOut, listDataname[j], selectedData)
483 return
483 return
484
484
485 def __selectDataMode2(self, data, blockIndex):
485 def __selectDataMode2(self, data, blockIndex):
486 utctime = data[:,0]
486 utctime = data[:,0]
487 aux, indices = numpy.unique(utctime, return_inverse=True)
487 aux, indices = numpy.unique(utctime, return_inverse=True)
488 selInd = numpy.where(indices == blockIndex)[0]
488 selInd = numpy.where(indices == blockIndex)[0]
489 selData = data[selInd,:]
489 selData = data[selInd,:]
490
490
491 return selData
491 return selData
492
492
493 def getData(self):
493 def getData(self):
494
494
495 # if self.flagNoMoreFiles:
495 # if self.flagNoMoreFiles:
496 # self.dataOut.flagNoData = True
496 # self.dataOut.flagNoData = True
497 # print 'Process finished'
497 # print 'Process finished'
498 # return 0
498 # return 0
499 #
499 #
500 if self.blockIndex==self.blocksPerFile:
500 if self.blockIndex==self.blocksPerFile:
501 if not( self.__setNextFileOffline() ):
501 if not( self.__setNextFileOffline() ):
502 self.dataOut.flagNoData = True
502 self.dataOut.flagNoData = True
503 return 0
503 return 0
504
504
505 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
505 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
506 # self.dataOut.flagNoData = True
506 # self.dataOut.flagNoData = True
507 # return 0
507 # return 0
508 # self.__readData()
508 # self.__readData()
509 self.__setDataOut()
509 self.__setDataOut()
510 self.dataOut.flagNoData = False
510 self.dataOut.flagNoData = False
511
511
512 self.blockIndex += 1
512 self.blockIndex += 1
513
513
514 return
514 return
515
515
516 def run(self, **kwargs):
516 def run(self, **kwargs):
517
517
518 if not(self.isConfig):
518 if not(self.isConfig):
519 self.setup(**kwargs)
519 self.setup(**kwargs)
520 # self.setObjProperties()
520 # self.setObjProperties()
521 self.isConfig = True
521 self.isConfig = True
522
522
523 self.getData()
523 self.getData()
524
524
525 return
525 return
526
526
527 class HDF5Writer(Operation):
527 class HDF5Writer(Operation):
528 '''
528 '''
529 HDF5 Writer, stores parameters data in HDF5 format files
529 HDF5 Writer, stores parameters data in HDF5 format files
530
530
531 path: path where the files will be stored
531 path: path where the files will be stored
532
532
533 blocksPerFile: number of blocks that will be saved in per HDF5 format file
533 blocksPerFile: number of blocks that will be saved in per HDF5 format file
534
534
535 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
535 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
536
536
537 metadataList: list of attributes that will be stored as metadata
537 metadataList: list of attributes that will be stored as metadata
538
538
539 dataList: list of attributes that will be stores as data
539 dataList: list of attributes that will be stores as data
540
540
541 '''
541 '''
542
542
543
543
544 ext = ".hdf5"
544 ext = ".hdf5"
545
545
546 optchar = "D"
546 optchar = "D"
547
547
548 metaoptchar = "M"
548 metaoptchar = "M"
549
549
550 metaFile = None
550 metaFile = None
551
551
552 filename = None
552 filename = None
553
553
554 path = None
554 path = None
555
555
556 setFile = None
556 setFile = None
557
557
558 fp = None
558 fp = None
559
559
560 grp = None
560 grp = None
561
561
562 ds = None
562 ds = None
563
563
564 firsttime = True
564 firsttime = True
565
565
566 #Configurations
566 #Configurations
567
567
568 blocksPerFile = None
568 blocksPerFile = None
569
569
570 blockIndex = None
570 blockIndex = None
571
571
572 dataOut = None
572 dataOut = None
573
573
574 #Data Arrays
574 #Data Arrays
575
575
576 dataList = None
576 dataList = None
577
577
578 metadataList = None
578 metadataList = None
579
579
580 # arrayDim = None
580 # arrayDim = None
581
581
582 dsList = None #List of dictionaries
582 dsList = None #List of dictionaries with dataset properties
583
583
584 tableDim = None
584 tableDim = None
585
585
586 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
586 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
587
587
588 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
588 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
589
589
590 mode = None
591
592 nDatas = None #Number of datasets to be stored per array
593
594 nDims = None #Number Dimensions in each dataset
595
596 nDimsForDs = None
597
598 currentDay = None
590 currentDay = None
599
591
600 def __init__(self):
592 def __init__(self):
601
593
602 Operation.__init__(self)
594 Operation.__init__(self)
603 self.isConfig = False
595 self.isConfig = False
604 return
596 return
605
597
606 def setup(self, dataOut, **kwargs):
598 def setup(self, dataOut, **kwargs):
607
599
608 self.path = kwargs['path']
600 self.path = kwargs['path']
609
601
610 if kwargs.has_key('blocksPerFile'):
602 if kwargs.has_key('blocksPerFile'):
611 self.blocksPerFile = kwargs['blocksPerFile']
603 self.blocksPerFile = kwargs['blocksPerFile']
612 else:
604 else:
613 self.blocksPerFile = 10
605 self.blocksPerFile = 10
614
606
615 self.metadataList = kwargs['metadataList']
607 self.metadataList = kwargs['metadataList']
616 self.dataList = kwargs['dataList']
608 self.dataList = kwargs['dataList']
617 self.dataOut = dataOut
609 self.dataOut = dataOut
618
610
619 if kwargs.has_key('mode'):
611 if kwargs.has_key('mode'):
620 mode = kwargs['mode']
612 mode = kwargs['mode']
621
613
622 if type(mode) == int:
614 if type(mode) == int:
623 mode = numpy.zeros(len(self.dataList)) + mode
615 mode = numpy.zeros(len(self.dataList)) + mode
624 else:
616 else:
625 mode = numpy.ones(len(self.dataList))
617 mode = numpy.ones(len(self.dataList))
626
618
627 self.mode = mode
619 self.mode = mode
628
620
629 arrayDim = numpy.zeros((len(self.dataList),5))
621 arrayDim = numpy.zeros((len(self.dataList),5))
630
622
631 #Table dimensions
623 #Table dimensions
632 dtype0 = self.dtype
624 dtype0 = self.dtype
633 tableList = []
625 tableList = []
634
626
635 #Dictionary and list of tables
627 #Dictionary and list of tables
636 dsList = []
628 dsList = []
637
629
638 for i in range(len(self.dataList)):
630 for i in range(len(self.dataList)):
639 dsDict = {}
631 dsDict = {}
640 dataAux = getattr(self.dataOut, self.dataList[i])
632 dataAux = getattr(self.dataOut, self.dataList[i])
641 dsDict['variable'] = self.dataList[i]
633 dsDict['variable'] = self.dataList[i]
642 #--------------------- Conditionals ------------------------
634 #--------------------- Conditionals ------------------------
643 #There is no data
635 #There is no data
644 if dataAux == None:
636 if dataAux == None:
645 return 0
637 return 0
646
638
647 #Not array, just a number
639 #Not array, just a number
648 #Mode 0
640 #Mode 0
649 if type(dataAux)==float or type(dataAux)==int:
641 if type(dataAux)==float or type(dataAux)==int:
650 dsDict['mode'] = 0
642 dsDict['mode'] = 0
651 dsDict['nDim'] = 0
643 dsDict['nDim'] = 0
652 arrayDim[i,0] = 1
644 arrayDim[i,0] = 0
653 dsList.append(dsDict)
645 dsList.append(dsDict)
654
646
655 #Mode 2: meteors
647 #Mode 2: meteors
656 elif mode[i] == 2:
648 elif mode[i] == 2:
657 # dsDict['nDim'] = 0
649 # dsDict['nDim'] = 0
658 dsDict['dsName'] = 'table0'
650 dsDict['dsName'] = 'table0'
659 dsDict['mode'] = 2 # Mode meteors
651 dsDict['mode'] = 2 # Mode meteors
660 dsDict['shape'] = dataAux.shape[-1]
652 dsDict['shape'] = dataAux.shape[-1]
661 dsDict['nDim'] = 0
653 dsDict['nDim'] = 0
654 dsDict['dsNumber'] = 1
662
655
663 arrayDim[i,3] = dataAux.shape[-1]
656 arrayDim[i,3] = dataAux.shape[-1]
664 arrayDim[i,4] = mode[i] #Mode the data was stored
657 arrayDim[i,4] = mode[i] #Mode the data was stored
665
658
666 dsList.append(dsDict)
659 dsList.append(dsDict)
667
660
668 #Mode 1
661 #Mode 1
669 else:
662 else:
670 arrayDim0 = dataAux.shape #Data dimensions
663 arrayDim0 = dataAux.shape #Data dimensions
671 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
664 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
672 arrayDim[i,4] = mode[i] #Mode the data was stored
665 arrayDim[i,4] = mode[i] #Mode the data was stored
673
666
674 strtable = 'table'
667 strtable = 'table'
675 dsDict['mode'] = 1 # Mode parameters
668 dsDict['mode'] = 1 # Mode parameters
676
669
677 # Three-dimension arrays
670 # Three-dimension arrays
678 if len(arrayDim0) == 3:
671 if len(arrayDim0) == 3:
679 arrayDim[i,1:-1] = numpy.array(arrayDim0)
672 arrayDim[i,1:-1] = numpy.array(arrayDim0)
680 nTables = int(arrayDim[i,2])
673 nTables = int(arrayDim[i,2])
681 dsDict['dsNumber'] = nTables
674 dsDict['dsNumber'] = nTables
682 dsDict['shape'] = arrayDim[i,2:4]
675 dsDict['shape'] = arrayDim[i,2:4]
683 dsDict['nDim'] = 3
676 dsDict['nDim'] = 3
684
677
685 for j in range(nTables):
678 for j in range(nTables):
686 dsDict = dsDict.copy()
679 dsDict = dsDict.copy()
687 dsDict['dsName'] = strtable + str(j)
680 dsDict['dsName'] = strtable + str(j)
688 dsList.append(dsDict)
681 dsList.append(dsDict)
689
682
690 # Two-dimension arrays
683 # Two-dimension arrays
691 elif len(arrayDim0) == 2:
684 elif len(arrayDim0) == 2:
692 arrayDim[i,2:-1] = numpy.array(arrayDim0)
685 arrayDim[i,2:-1] = numpy.array(arrayDim0)
693 nTables = int(arrayDim[i,2])
686 nTables = int(arrayDim[i,2])
694 dsDict['dsNumber'] = nTables
687 dsDict['dsNumber'] = nTables
695 dsDict['shape'] = arrayDim[i,3]
688 dsDict['shape'] = arrayDim[i,3]
696 dsDict['nDim'] = 2
689 dsDict['nDim'] = 2
697
690
698 for j in range(nTables):
691 for j in range(nTables):
699 dsDict = dsDict.copy()
692 dsDict = dsDict.copy()
700 dsDict['dsName'] = strtable + str(j)
693 dsDict['dsName'] = strtable + str(j)
701 dsList.append(dsDict)
694 dsList.append(dsDict)
702
695
703 # One-dimension arrays
696 # One-dimension arrays
704 elif len(arrayDim0) == 1:
697 elif len(arrayDim0) == 1:
705 arrayDim[i,3] = arrayDim0[0]
698 arrayDim[i,3] = arrayDim0[0]
706 dsDict['shape'] = arrayDim0[0]
699 dsDict['shape'] = arrayDim0[0]
707 dsDict['dsNumber'] = 1
700 dsDict['dsNumber'] = 1
708 dsDict['dsName'] = strtable + str(0)
701 dsDict['dsName'] = strtable + str(0)
709 dsDict['nDim'] = 1
702 dsDict['nDim'] = 1
710 dsList.append(dsDict)
703 dsList.append(dsDict)
711
704
712 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
705 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
713 tableList.append(table)
706 tableList.append(table)
714
707
715 # self.arrayDim = arrayDim
708 # self.arrayDim = arrayDim
716 self.dsList = dsList
709 self.dsList = dsList
717 self.tableDim = numpy.array(tableList, dtype = dtype0)
710 self.tableDim = numpy.array(tableList, dtype = dtype0)
718 self.blockIndex = 0
711 self.blockIndex = 0
719
712
720 timeTuple = time.localtime(dataOut.utctime)
713 timeTuple = time.localtime(dataOut.utctime)
721 self.currentDay = timeTuple.tm_yday
714 self.currentDay = timeTuple.tm_yday
722 return 1
715 return 1
723
716
724 def putMetadata(self):
717 def putMetadata(self):
725
718
726 fp = self.createMetadataFile()
719 fp = self.createMetadataFile()
727 self.writeMetadata(fp)
720 self.writeMetadata(fp)
728 fp.close()
721 fp.close()
729 return
722 return
730
723
731 def createMetadataFile(self):
724 def createMetadataFile(self):
732 ext = self.ext
725 ext = self.ext
733 path = self.path
726 path = self.path
734 setFile = self.setFile
727 setFile = self.setFile
735
728
736 timeTuple = time.localtime(self.dataOut.utctime)
729 timeTuple = time.localtime(self.dataOut.utctime)
737
730
738 subfolder = ''
731 subfolder = ''
739 fullpath = os.path.join( path, subfolder )
732 fullpath = os.path.join( path, subfolder )
740
733
741 if not( os.path.exists(fullpath) ):
734 if not( os.path.exists(fullpath) ):
742 os.mkdir(fullpath)
735 os.mkdir(fullpath)
743 setFile = -1 #inicializo mi contador de seteo
736 setFile = -1 #inicializo mi contador de seteo
744
737
745 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
738 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
746 fullpath = os.path.join( path, subfolder )
739 fullpath = os.path.join( path, subfolder )
747
740
748 if not( os.path.exists(fullpath) ):
741 if not( os.path.exists(fullpath) ):
749 os.mkdir(fullpath)
742 os.mkdir(fullpath)
750 setFile = -1 #inicializo mi contador de seteo
743 setFile = -1 #inicializo mi contador de seteo
751
744
752 else:
745 else:
753 filesList = os.listdir( fullpath )
746 filesList = os.listdir( fullpath )
754 filesList = sorted( filesList, key=str.lower )
747 filesList = sorted( filesList, key=str.lower )
755 if len( filesList ) > 0:
748 if len( filesList ) > 0:
756 filesList = [k for k in filesList if 'M' in k]
749 filesList = [k for k in filesList if 'M' in k]
757 filen = filesList[-1]
750 filen = filesList[-1]
758 # el filename debera tener el siguiente formato
751 # el filename debera tener el siguiente formato
759 # 0 1234 567 89A BCDE (hex)
752 # 0 1234 567 89A BCDE (hex)
760 # x YYYY DDD SSS .ext
753 # x YYYY DDD SSS .ext
761 if isNumber( filen[8:11] ):
754 if isNumber( filen[8:11] ):
762 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
755 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
763 else:
756 else:
764 setFile = -1
757 setFile = -1
765 else:
758 else:
766 setFile = -1 #inicializo mi contador de seteo
759 setFile = -1 #inicializo mi contador de seteo
767
760
768 setFile += 1
761 setFile += 1
769
762
770 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
763 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
771 timeTuple.tm_year,
764 timeTuple.tm_year,
772 timeTuple.tm_yday,
765 timeTuple.tm_yday,
773 setFile,
766 setFile,
774 ext )
767 ext )
775
768
776 filename = os.path.join( path, subfolder, file )
769 filename = os.path.join( path, subfolder, file )
777 self.metaFile = file
770 self.metaFile = file
778 #Setting HDF5 File
771 #Setting HDF5 File
779 fp = h5py.File(filename,'w')
772 fp = h5py.File(filename,'w')
780
773
781 return fp
774 return fp
782
775
783 def writeMetadata(self, fp):
776 def writeMetadata(self, fp):
784
777
785 grp = fp.create_group("Metadata")
778 grp = fp.create_group("Metadata")
786 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
779 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
787
780
788 for i in range(len(self.metadataList)):
781 for i in range(len(self.metadataList)):
789 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
782 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
790 return
783 return
791
784
792 def dateFlag(self):
785 def dateFlag(self):
793
786
794 timeTuple = time.localtime(self.dataOut.utctime)
787 timeTuple = time.localtime(self.dataOut.utctime)
795 dataDay = timeTuple.tm_yday
788 dataDay = timeTuple.tm_yday
796
789
797 if dataDay == self.currentDay:
790 if dataDay == self.currentDay:
798 return False
791 return False
799
792
800 self.currentDay = dataDay
793 self.currentDay = dataDay
801 return True
794 return True
802
795
803 def setNextFile(self):
796 def setNextFile(self):
804
797
805 ext = self.ext
798 ext = self.ext
806 path = self.path
799 path = self.path
807 setFile = self.setFile
800 setFile = self.setFile
808 mode = self.mode
801 mode = self.mode
809
802
810 timeTuple = time.localtime(self.dataOut.utctime)
803 timeTuple = time.localtime(self.dataOut.utctime)
811 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
804 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
812
805
813 fullpath = os.path.join( path, subfolder )
806 fullpath = os.path.join( path, subfolder )
814
807
815 if os.path.exists(fullpath):
808 if os.path.exists(fullpath):
816 filesList = os.listdir( fullpath )
809 filesList = os.listdir( fullpath )
817 filesList = [k for k in filesList if 'D' in k]
810 filesList = [k for k in filesList if 'D' in k]
818 if len( filesList ) > 0:
811 if len( filesList ) > 0:
819 filesList = sorted( filesList, key=str.lower )
812 filesList = sorted( filesList, key=str.lower )
820 filen = filesList[-1]
813 filen = filesList[-1]
821 # el filename debera tener el siguiente formato
814 # el filename debera tener el siguiente formato
822 # 0 1234 567 89A BCDE (hex)
815 # 0 1234 567 89A BCDE (hex)
823 # x YYYY DDD SSS .ext
816 # x YYYY DDD SSS .ext
824 if isNumber( filen[8:11] ):
817 if isNumber( filen[8:11] ):
825 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
818 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
826 else:
819 else:
827 setFile = -1
820 setFile = -1
828 else:
821 else:
829 setFile = -1 #inicializo mi contador de seteo
822 setFile = -1 #inicializo mi contador de seteo
830 else:
823 else:
831 os.mkdir(fullpath)
824 os.mkdir(fullpath)
832 setFile = -1 #inicializo mi contador de seteo
825 setFile = -1 #inicializo mi contador de seteo
833
826
834 setFile += 1
827 setFile += 1
835
828
836 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
829 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
837 timeTuple.tm_year,
830 timeTuple.tm_year,
838 timeTuple.tm_yday,
831 timeTuple.tm_yday,
839 setFile,
832 setFile,
840 ext )
833 ext )
841
834
842 filename = os.path.join( path, subfolder, file )
835 filename = os.path.join( path, subfolder, file )
843
836
844 #Setting HDF5 File
837 #Setting HDF5 File
845 fp = h5py.File(filename,'w')
838 fp = h5py.File(filename,'w')
846 #write metadata
839 #write metadata
847 self.writeMetadata(fp)
840 self.writeMetadata(fp)
848 #Write data
841 #Write data
849 grp = fp.create_group("Data")
842 grp = fp.create_group("Data")
850 # grp.attrs['metadata'] = self.metaFile
843 # grp.attrs['metadata'] = self.metaFile
851
844
852 # grp.attrs['blocksPerFile'] = 0
845 # grp.attrs['blocksPerFile'] = 0
853 ds = []
846 ds = []
854 data = []
847 data = []
855 dsList = self.dsList
848 dsList = self.dsList
856 i = 0
849 i = 0
857 while i < len(dsList):
850 while i < len(dsList):
858 dsInfo = dsList[i]
851 dsInfo = dsList[i]
859 #One-dimension data
852 #One-dimension data
860 if dsInfo['mode'] == 0:
853 if dsInfo['mode'] == 0:
861 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
854 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
862 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
855 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
863 ds.append(ds0)
856 ds.append(ds0)
864 data.append([])
857 data.append([])
865 i += 1
858 i += 1
866 continue
859 continue
867 # nDimsForDs.append(nDims[i])
860 # nDimsForDs.append(nDims[i])
868
861
869 elif dsInfo['mode'] == 2:
862 elif dsInfo['mode'] == 2:
870 grp0 = grp.create_group(dsInfo['variable'])
863 grp0 = grp.create_group(dsInfo['variable'])
871 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
864 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
872 ds.append(ds0)
865 ds.append(ds0)
873 data.append([])
866 data.append([])
874 i += 1
867 i += 1
875 continue
868 continue
876
869
877 elif dsInfo['mode'] == 1:
870 elif dsInfo['mode'] == 1:
878 grp0 = grp.create_group(dsInfo['variable'])
871 grp0 = grp.create_group(dsInfo['variable'])
879
872
880 for j in range(dsInfo['dsNumber']):
873 for j in range(dsInfo['dsNumber']):
881 dsInfo = dsList[i]
874 dsInfo = dsList[i]
882 tableName = dsInfo['dsName']
875 tableName = dsInfo['dsName']
883 shape = dsInfo['shape']
876 shape = dsInfo['shape']
884
877
885 if dsInfo['nDim'] == 3:
878 if dsInfo['nDim'] == 3:
886 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
879 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
887 else:
880 else:
888 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
881 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
889
882
890 ds.append(ds0)
883 ds.append(ds0)
891 data.append([])
884 data.append([])
892 i += 1
885 i += 1
893 # nDimsForDs.append(nDims[i])
886 # nDimsForDs.append(nDims[i])
894
887
895 fp.flush()
888 fp.flush()
896 fp.close()
889 fp.close()
897
890
898 # self.nDatas = nDatas
891 # self.nDatas = nDatas
899 # self.nDims = nDims
892 # self.nDims = nDims
900 # self.nDimsForDs = nDimsForDs
893 # self.nDimsForDs = nDimsForDs
901 #Saving variables
894 #Saving variables
902 print 'Writing the file: %s'%filename
895 print 'Writing the file: %s'%filename
903 self.filename = filename
896 self.filename = filename
904 # self.fp = fp
897 # self.fp = fp
905 # self.grp = grp
898 # self.grp = grp
906 # self.grp.attrs.modify('nRecords', 1)
899 # self.grp.attrs.modify('nRecords', 1)
907 self.ds = ds
900 self.ds = ds
908 self.data = data
901 self.data = data
909 # self.setFile = setFile
902 # self.setFile = setFile
910 self.firsttime = True
903 self.firsttime = True
911 self.blockIndex = 0
904 self.blockIndex = 0
912 return
905 return
913
906
914 def setNextFile1(self):
915
916 ext = self.ext
917 path = self.path
918 setFile = self.setFile
919 mode = self.mode
920
921 timeTuple = time.localtime(self.dataOut.utctime)
922 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
923
924 fullpath = os.path.join( path, subfolder )
925
926 if os.path.exists(fullpath):
927 filesList = os.listdir( fullpath )
928 filesList = [k for k in filesList if 'D' in k]
929 if len( filesList ) > 0:
930 filesList = sorted( filesList, key=str.lower )
931 filen = filesList[-1]
932 # el filename debera tener el siguiente formato
933 # 0 1234 567 89A BCDE (hex)
934 # x YYYY DDD SSS .ext
935 if isNumber( filen[8:11] ):
936 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
937 else:
938 setFile = -1
939 else:
940 setFile = -1 #inicializo mi contador de seteo
941 else:
942 os.mkdir(fullpath)
943 setFile = -1 #inicializo mi contador de seteo
944
945 setFile += 1
946
947 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
948 timeTuple.tm_year,
949 timeTuple.tm_yday,
950 setFile,
951 ext )
952
953 filename = os.path.join( path, subfolder, file )
954
955 #Setting HDF5 File
956 fp = h5py.File(filename,'w')
957
958 #writemetadata
959 self.writeMetadata(fp)
960
961 grp = fp.create_group("Data")
962 # grp.attrs['metadata'] = self.metaFile
963
964 # grp.attrs['blocksPerFile'] = 0
965
966 ds = []
967 data = []
968 nDimsForDs = []
969
970 nDatas = numpy.zeros(len(self.dataList))
971 nDims = self.arrayDim[:,0]
972
973 nDim1 = self.arrayDim[:,2]
974 nDim0 = self.arrayDim[:,3]
975
976 for i in range(len(self.dataList)):
977
978 #One-dimension data
979 if nDims[i]==1:
980 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
981 ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
982 ds.append(ds0)
983 data.append([])
984 nDimsForDs.append(nDims[i])
985 else:
986
987 #Channel mode
988 # if mode[i] == 0:
989 # strMode = "channel"
990 #
991 # #nDatas is the number of arrays per variable
992 # if nDims[i] == 1:
993 # nDatas[i] = self.arrayDim[i,1]
994 # elif nDims[i] == 2:
995 # nDatas[i] = self.arrayDim[i,2]
996
997 #Parameters mode
998 if mode[i] == 1:
999 strMode = "param"
1000 nDatas[i] = self.arrayDim[i,2]
1001
1002 #Meteors mode
1003 elif mode[i] == 2:
1004 strMode = "table"
1005 nDatas[i] = 1
1006
1007 grp0 = grp.create_group(self.dataList[i])
1008
1009 for j in range(int(nDatas[i])):
1010 tableName = strMode + str(j)
1011
1012 if nDims[i] == 3:
1013 ds0 = grp0.create_dataset(tableName, (nDim1[i],nDim0[i],1) , data = numpy.zeros((nDim1[i],nDim0[i],1)) ,maxshape=(None,nDim0[i],None), chunks=True)
1014
1015 else:
1016 ds0 = grp0.create_dataset(tableName, (1,nDim0[i]), data = numpy.zeros((1,nDim0[i])) , maxshape=(None,nDim0[i]), chunks=True)
1017
1018 ds.append(ds0)
1019 data.append([])
1020 nDimsForDs.append(nDims[i])
1021
1022 fp.flush()
1023 fp.close()
1024
1025 self.nDatas = nDatas
1026 self.nDims = nDims
1027 self.nDimsForDs = nDimsForDs
1028 #Saving variables
1029 print 'Writing the file: %s'%filename
1030 self.filename = filename
1031 # self.fp = fp
1032 # self.grp = grp
1033 # self.grp.attrs.modify('nRecords', 1)
1034 self.ds = ds
1035 self.data = data
1036 #
1037 # self.setFile = setFile
1038 self.firsttime = True
1039 self.blockIndex = 0
1040 return
1041
1042 def putData(self):
907 def putData(self):
1043
908
1044 if self.blockIndex == self.blocksPerFile or self.dateFlag():
909 if self.blockIndex == self.blocksPerFile or self.dateFlag():
1045 self.setNextFile()
910 self.setNextFile()
1046
911
1047 # if not self.firsttime:
912 # if not self.firsttime:
1048 self.readBlock()
913 self.readBlock()
1049 self.setBlock() #Prepare data to be written
914 self.setBlock() #Prepare data to be written
1050 self.writeBlock() #Write data
915 self.writeBlock() #Write data
1051
916
1052 return
917 return
1053
918
1054 def readBlock(self):
919 def readBlock(self):
1055
920
1056 '''
921 '''
1057 data Array configured
922 data Array configured
1058
923
1059
924
1060 self.data
925 self.data
1061 '''
926 '''
1062 dsList = self.dsList
927 dsList = self.dsList
1063 ds = self.ds
928 ds = self.ds
1064 #Setting HDF5 File
929 #Setting HDF5 File
1065 fp = h5py.File(self.filename,'r+')
930 fp = h5py.File(self.filename,'r+')
1066 grp = fp["Data"]
931 grp = fp["Data"]
1067 ind = 0
932 ind = 0
1068
933
1069 # grp.attrs['blocksPerFile'] = 0
934 # grp.attrs['blocksPerFile'] = 0
1070 while ind < len(dsList):
935 while ind < len(dsList):
1071 dsInfo = dsList[ind]
936 dsInfo = dsList[ind]
1072
937
1073 if dsInfo['mode'] == 0:
938 if dsInfo['mode'] == 0:
1074 ds0 = grp[dsInfo['variable']]
939 ds0 = grp[dsInfo['variable']]
1075 ds[ind] = ds0
940 ds[ind] = ds0
1076 ind += 1
941 ind += 1
1077 else:
942 else:
1078
943
1079 grp0 = grp[dsInfo['variable']]
944 grp0 = grp[dsInfo['variable']]
1080
945
1081 for j in range(dsInfo['dsNumber']):
946 for j in range(dsInfo['dsNumber']):
1082 dsInfo = dsList[ind]
947 dsInfo = dsList[ind]
1083 ds0 = grp0[dsInfo['dsName']]
948 ds0 = grp0[dsInfo['dsName']]
1084 ds[ind] = ds0
949 ds[ind] = ds0
1085 ind += 1
950 ind += 1
1086
951
1087 self.fp = fp
952 self.fp = fp
1088 self.grp = grp
953 self.grp = grp
1089 self.ds = ds
954 self.ds = ds
1090
955
1091 return
956 return
1092
957
1093 def setBlock(self):
958 def setBlock(self):
1094 '''
959 '''
1095 data Array configured
960 data Array configured
1096
961
1097
962
1098 self.data
963 self.data
1099 '''
964 '''
1100 #Creating Arrays
965 #Creating Arrays
1101 dsList = self.dsList
966 dsList = self.dsList
1102 data = self.data
967 data = self.data
1103 ind = 0
968 ind = 0
1104
969
1105 while ind < len(dsList):
970 while ind < len(dsList):
1106 dsInfo = dsList[ind]
971 dsInfo = dsList[ind]
1107 dataAux = getattr(self.dataOut, dsInfo['variable'])
972 dataAux = getattr(self.dataOut, dsInfo['variable'])
1108
973
1109 mode = dsInfo['mode']
974 mode = dsInfo['mode']
1110 nDim = dsInfo['nDim']
975 nDim = dsInfo['nDim']
1111
976
1112 if mode == 0 or mode == 2 or nDim == 1:
977 if mode == 0 or mode == 2 or nDim == 1:
1113 data[ind] = dataAux
978 data[ind] = dataAux
1114 ind += 1
979 ind += 1
1115 # elif nDim == 1:
980 # elif nDim == 1:
1116 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
981 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1117 # ind += 1
982 # ind += 1
1118 elif nDim == 2:
983 elif nDim == 2:
1119 for j in range(dsInfo['dsNumber']):
984 for j in range(dsInfo['dsNumber']):
1120 data[ind] = dataAux[j,:]
985 data[ind] = dataAux[j,:]
1121 ind += 1
986 ind += 1
1122 elif nDim == 3:
987 elif nDim == 3:
1123 for j in range(dsInfo['dsNumber']):
988 for j in range(dsInfo['dsNumber']):
1124 data[ind] = dataAux[:,j,:]
989 data[ind] = dataAux[:,j,:]
1125 ind += 1
990 ind += 1
1126
991
1127 self.data = data
992 self.data = data
1128 return
993 return
1129
994
1130 def writeBlock(self):
995 def writeBlock(self):
1131 '''
996 '''
1132 Saves the block in the HDF5 file
997 Saves the block in the HDF5 file
1133 '''
998 '''
1134 dsList = self.dsList
999 dsList = self.dsList
1135
1000
1136 for i in range(len(self.ds)):
1001 for i in range(len(self.ds)):
1137 dsInfo = dsList[i]
1002 dsInfo = dsList[i]
1138 nDim = dsInfo['nDim']
1003 nDim = dsInfo['nDim']
1139 mode = dsInfo['mode']
1004 mode = dsInfo['mode']
1140
1005
1141 # First time
1006 # First time
1142 if self.firsttime:
1007 if self.firsttime:
1143 # self.ds[i].resize(self.data[i].shape)
1008 # self.ds[i].resize(self.data[i].shape)
1144 # self.ds[i][self.blockIndex,:] = self.data[i]
1009 # self.ds[i][self.blockIndex,:] = self.data[i]
1145 if type(self.data[i]) == numpy.ndarray:
1010 if type(self.data[i]) == numpy.ndarray:
1146
1011
1147 if nDim == 3:
1012 if nDim == 3:
1148 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1013 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1149 self.ds[i].resize(self.data[i].shape)
1014 self.ds[i].resize(self.data[i].shape)
1150
1015 if mode == 2:
1016 self.ds[i].resize(self.data[i].shape)
1151 self.ds[i][:] = self.data[i]
1017 self.ds[i][:] = self.data[i]
1152 else:
1018 else:
1153
1019
1154 # From second time
1020 # From second time
1155 # Meteors!
1021 # Meteors!
1156 if mode == 2:
1022 if mode == 2:
1157 dataShape = self.data[i].shape
1023 dataShape = self.data[i].shape
1158 dsShape = self.ds[i].shape
1024 dsShape = self.ds[i].shape
1159 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1025 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1160 self.ds[i][dsShape[0]:,:] = self.data[i]
1026 self.ds[i][dsShape[0]:,:] = self.data[i]
1161 # No dimension
1027 # No dimension
1162 elif mode == 0:
1028 elif mode == 0:
1163 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1029 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1164 self.ds[i][0,-1] = self.data[i]
1030 self.ds[i][0,-1] = self.data[i]
1165 # One dimension
1031 # One dimension
1166 elif nDim == 1:
1032 elif nDim == 1:
1167 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1033 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1168 self.ds[i][-1,:] = self.data[i]
1034 self.ds[i][-1,:] = self.data[i]
1169 # Two dimension
1035 # Two dimension
1170 elif nDim == 2:
1036 elif nDim == 2:
1171 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1037 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1172 self.ds[i][self.blockIndex,:] = self.data[i]
1038 self.ds[i][self.blockIndex,:] = self.data[i]
1173 # Three dimensions
1039 # Three dimensions
1174 elif nDim == 3:
1040 elif nDim == 3:
1175 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1041 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1176 self.ds[i][:,:,-1] = self.data[i]
1042 self.ds[i][:,:,-1] = self.data[i]
1177
1043
1178 self.firsttime = False
1044 self.firsttime = False
1179 self.blockIndex += 1
1045 self.blockIndex += 1
1180
1046
1181 #Close to save changes
1047 #Close to save changes
1182 self.fp.flush()
1048 self.fp.flush()
1183 self.fp.close()
1049 self.fp.close()
1184 return
1050 return
1185
1051
1186 def run(self, dataOut, **kwargs):
1052 def run(self, dataOut, **kwargs):
1187
1053
1188 if not(self.isConfig):
1054 if not(self.isConfig):
1189 flagdata = self.setup(dataOut, **kwargs)
1055 flagdata = self.setup(dataOut, **kwargs)
1190
1056
1191 if not(flagdata):
1057 if not(flagdata):
1192 return
1058 return
1193
1059
1194 self.isConfig = True
1060 self.isConfig = True
1195 # self.putMetadata()
1061 # self.putMetadata()
1196 self.setNextFile()
1062 self.setNextFile()
1197
1063
1198 self.putData()
1064 self.putData()
1199 return
1065 return
1200
1066
1201
1067
General Comments 0
You need to be logged in to leave comments. Login now