##// END OF EJS Templates
schain-jc
RSS
Bug in ParamWriter not saving numpy types (int, float)
Juan C. Espinoza -
r1195:9f1881ac55a3
parent child
Show More
Side by Side Unified
Context file:
r1195:9f1881ac55a3 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1036 +1,1035
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 # from .jroIO_base import *
10 # from .jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13 from schainpy.utils import log
13 from schainpy.utils import log
14
14
15 @MPDecorator
15 @MPDecorator
16 class ParamReader(JRODataReader,ProcessingUnit):
16 class ParamReader(JRODataReader,ProcessingUnit):
17 '''
17 '''
18 Reads HDF5 format files
18 Reads HDF5 format files
19
19
20 path
20 path
21
21
22 startDate
22 startDate
23
23
24 endDate
24 endDate
25
25
26 startTime
26 startTime
27
27
28 endTime
28 endTime
29 '''
29 '''
30
30
31 ext = ".hdf5"
31 ext = ".hdf5"
32
32
33 optchar = "D"
33 optchar = "D"
34
34
35 timezone = None
35 timezone = None
36
36
37 startTime = None
37 startTime = None
38
38
39 endTime = None
39 endTime = None
40
40
41 fileIndex = None
41 fileIndex = None
42
42
43 utcList = None #To select data in the utctime list
43 utcList = None #To select data in the utctime list
44
44
45 blockList = None #List to blocks to be read from the file
45 blockList = None #List to blocks to be read from the file
46
46
47 blocksPerFile = None #Number of blocks to be read
47 blocksPerFile = None #Number of blocks to be read
48
48
49 blockIndex = None
49 blockIndex = None
50
50
51 path = None
51 path = None
52
52
53 #List of Files
53 #List of Files
54
54
55 filenameList = None
55 filenameList = None
56
56
57 datetimeList = None
57 datetimeList = None
58
58
59 #Hdf5 File
59 #Hdf5 File
60
60
61 listMetaname = None
61 listMetaname = None
62
62
63 listMeta = None
63 listMeta = None
64
64
65 listDataname = None
65 listDataname = None
66
66
67 listData = None
67 listData = None
68
68
69 listShapes = None
69 listShapes = None
70
70
71 fp = None
71 fp = None
72
72
73 #dataOut reconstruction
73 #dataOut reconstruction
74
74
75 dataOut = None
75 dataOut = None
76
76
77
77
78 def __init__(self):#, **kwargs):
78 def __init__(self):#, **kwargs):
79 ProcessingUnit.__init__(self) #, **kwargs)
79 ProcessingUnit.__init__(self) #, **kwargs)
80 self.dataOut = Parameters()
80 self.dataOut = Parameters()
81 return
81 return
82
82
83 def setup(self, **kwargs):
83 def setup(self, **kwargs):
84
84
85 path = kwargs['path']
85 path = kwargs['path']
86 startDate = kwargs['startDate']
86 startDate = kwargs['startDate']
87 endDate = kwargs['endDate']
87 endDate = kwargs['endDate']
88 startTime = kwargs['startTime']
88 startTime = kwargs['startTime']
89 endTime = kwargs['endTime']
89 endTime = kwargs['endTime']
90 walk = kwargs['walk']
90 walk = kwargs['walk']
91 if 'ext' in kwargs:
91 if 'ext' in kwargs:
92 ext = kwargs['ext']
92 ext = kwargs['ext']
93 else:
93 else:
94 ext = '.hdf5'
94 ext = '.hdf5'
95 if 'timezone' in kwargs:
95 if 'timezone' in kwargs:
96 self.timezone = kwargs['timezone']
96 self.timezone = kwargs['timezone']
97 else:
97 else:
98 self.timezone = 'lt'
98 self.timezone = 'lt'
99
99
100 print("[Reading] Searching files in offline mode ...")
100 print("[Reading] Searching files in offline mode ...")
101 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
102 startTime=startTime, endTime=endTime,
102 startTime=startTime, endTime=endTime,
103 ext=ext, walk=walk)
103 ext=ext, walk=walk)
104
104
105 if not(filenameList):
105 if not(filenameList):
106 print("There is no files into the folder: %s"%(path))
106 print("There is no files into the folder: %s"%(path))
107 sys.exit(-1)
107 sys.exit(-1)
108
108
109 self.fileIndex = -1
109 self.fileIndex = -1
110 self.startTime = startTime
110 self.startTime = startTime
111 self.endTime = endTime
111 self.endTime = endTime
112
112
113 self.__readMetadata()
113 self.__readMetadata()
114
114
115 self.__setNextFileOffline()
115 self.__setNextFileOffline()
116
116
117 return
117 return
118
118
119 def searchFilesOffLine(self,
119 def searchFilesOffLine(self,
120 path,
120 path,
121 startDate=None,
121 startDate=None,
122 endDate=None,
122 endDate=None,
123 startTime=datetime.time(0,0,0),
123 startTime=datetime.time(0,0,0),
124 endTime=datetime.time(23,59,59),
124 endTime=datetime.time(23,59,59),
125 ext='.hdf5',
125 ext='.hdf5',
126 walk=True):
126 walk=True):
127
127
128 expLabel = ''
128 expLabel = ''
129 self.filenameList = []
129 self.filenameList = []
130 self.datetimeList = []
130 self.datetimeList = []
131
131
132 pathList = []
132 pathList = []
133
133
134 JRODataObj = JRODataReader()
134 JRODataObj = JRODataReader()
135 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
136
136
137 if dateList == []:
137 if dateList == []:
138 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
139 datetime.datetime.combine(startDate,startTime).ctime(),
139 datetime.datetime.combine(startDate,startTime).ctime(),
140 datetime.datetime.combine(endDate,endTime).ctime()))
140 datetime.datetime.combine(endDate,endTime).ctime()))
141
141
142 return None, None
142 return None, None
143
143
144 if len(dateList) > 1:
144 if len(dateList) > 1:
145 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
145 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
146 else:
146 else:
147 print("[Reading] data was found for the date %s" %(dateList[0]))
147 print("[Reading] data was found for the date %s" %(dateList[0]))
148
148
149 filenameList = []
149 filenameList = []
150 datetimeList = []
150 datetimeList = []
151
151
152 #----------------------------------------------------------------------------------
152 #----------------------------------------------------------------------------------
153
153
154 for thisPath in pathList:
154 for thisPath in pathList:
155 # thisPath = pathList[pathDict[file]]
155 # thisPath = pathList[pathDict[file]]
156
156
157 fileList = glob.glob1(thisPath, "*%s" %ext)
157 fileList = glob.glob1(thisPath, "*%s" %ext)
158 fileList.sort()
158 fileList.sort()
159
159
160 for file in fileList:
160 for file in fileList:
161
161
162 filename = os.path.join(thisPath,file)
162 filename = os.path.join(thisPath,file)
163
163
164 if not isFileInDateRange(filename, startDate, endDate):
164 if not isFileInDateRange(filename, startDate, endDate):
165 continue
165 continue
166
166
167 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
168
168
169 if not(thisDatetime):
169 if not(thisDatetime):
170 continue
170 continue
171
171
172 filenameList.append(filename)
172 filenameList.append(filename)
173 datetimeList.append(thisDatetime)
173 datetimeList.append(thisDatetime)
174
174
175 if not(filenameList):
175 if not(filenameList):
176 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
176 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
177 return None, None
177 return None, None
178
178
179 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
179 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
180 print()
180 print()
181
181
182 self.filenameList = filenameList
182 self.filenameList = filenameList
183 self.datetimeList = datetimeList
183 self.datetimeList = datetimeList
184
184
185 return pathList, filenameList
185 return pathList, filenameList
186
186
187 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
187 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
188
188
189 """
189 """
190 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
190 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
191
191
192 Inputs:
192 Inputs:
193 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
193 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
194
194
195 startDate : fecha inicial del rango seleccionado en formato datetime.date
195 startDate : fecha inicial del rango seleccionado en formato datetime.date
196
196
197 endDate : fecha final del rango seleccionado en formato datetime.date
197 endDate : fecha final del rango seleccionado en formato datetime.date
198
198
199 startTime : tiempo inicial del rango seleccionado en formato datetime.time
199 startTime : tiempo inicial del rango seleccionado en formato datetime.time
200
200
201 endTime : tiempo final del rango seleccionado en formato datetime.time
201 endTime : tiempo final del rango seleccionado en formato datetime.time
202
202
203 Return:
203 Return:
204 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
204 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
205 fecha especificado, de lo contrario retorna False.
205 fecha especificado, de lo contrario retorna False.
206
206
207 Excepciones:
207 Excepciones:
208 Si el archivo no existe o no puede ser abierto
208 Si el archivo no existe o no puede ser abierto
209 Si la cabecera no puede ser leida.
209 Si la cabecera no puede ser leida.
210
210
211 """
211 """
212
212
213 try:
213 try:
214 fp = h5py.File(filename,'r')
214 fp = h5py.File(filename,'r')
215 grp1 = fp['Data']
215 grp1 = fp['Data']
216
216
217 except IOError:
217 except IOError:
218 traceback.print_exc()
218 traceback.print_exc()
219 raise IOError("The file %s can't be opened" %(filename))
219 raise IOError("The file %s can't be opened" %(filename))
220 #chino rata
220 #chino rata
221 #In case has utctime attribute
221 #In case has utctime attribute
222 grp2 = grp1['utctime']
222 grp2 = grp1['utctime']
223 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
223 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
224 thisUtcTime = grp2.value[0]
224 thisUtcTime = grp2.value[0]
225
225
226 fp.close()
226 fp.close()
227
227
228 if self.timezone == 'lt':
228 if self.timezone == 'lt':
229 thisUtcTime -= 5*3600
229 thisUtcTime -= 5*3600
230
230
231 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
231 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
232 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
232 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
233 thisDate = thisDatetime.date()
233 thisDate = thisDatetime.date()
234 thisTime = thisDatetime.time()
234 thisTime = thisDatetime.time()
235
235
236 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
236 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
237 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
237 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238
238
239 #General case
239 #General case
240 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
240 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
241 #-----------o----------------------------o-----------
241 #-----------o----------------------------o-----------
242 # startTime endTime
242 # startTime endTime
243
243
244 if endTime >= startTime:
244 if endTime >= startTime:
245 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
245 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
246 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
246 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
247 return thisDatetime
247 return thisDatetime
248 return None
248 return None
249
249
250 #If endTime < startTime then endTime belongs to the next day
250 #If endTime < startTime then endTime belongs to the next day
251 #<<<<<<<<<<<o o>>>>>>>>>>>
251 #<<<<<<<<<<<o o>>>>>>>>>>>
252 #-----------o----------------------------o-----------
252 #-----------o----------------------------o-----------
253 # endTime startTime
253 # endTime startTime
254
254
255 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
255 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
256 return None
256 return None
257
257
258 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
258 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
259 return None
259 return None
260
260
261 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
261 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
262 return None
262 return None
263
263
264 return thisDatetime
264 return thisDatetime
265
265
266 def __setNextFileOffline(self):
266 def __setNextFileOffline(self):
267
267
268 self.fileIndex += 1
268 self.fileIndex += 1
269 idFile = self.fileIndex
269 idFile = self.fileIndex
270
270
271 if not(idFile < len(self.filenameList)):
271 if not(idFile < len(self.filenameList)):
272 print("No more Files")
272 print("No more Files")
273 return 0
273 return 0
274
274
275 filename = self.filenameList[idFile]
275 filename = self.filenameList[idFile]
276
276
277 filePointer = h5py.File(filename,'r')
277 filePointer = h5py.File(filename,'r')
278
278
279 self.filename = filename
279 self.filename = filename
280
280
281 self.fp = filePointer
281 self.fp = filePointer
282
282
283 print("Setting the file: %s"%self.filename)
283 print("Setting the file: %s"%self.filename)
284
284
285 # self.__readMetadata()
285 # self.__readMetadata()
286 self.__setBlockList()
286 self.__setBlockList()
287 self.__readData()
287 self.__readData()
288 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
288 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
289 # self.nRecords = self.fp['Data'].attrs['nRecords']
289 # self.nRecords = self.fp['Data'].attrs['nRecords']
290 self.blockIndex = 0
290 self.blockIndex = 0
291 return 1
291 return 1
292
292
293 def __setBlockList(self):
293 def __setBlockList(self):
294 '''
294 '''
295 Selects the data within the times defined
295 Selects the data within the times defined
296
296
297 self.fp
297 self.fp
298 self.startTime
298 self.startTime
299 self.endTime
299 self.endTime
300
300
301 self.blockList
301 self.blockList
302 self.blocksPerFile
302 self.blocksPerFile
303
303
304 '''
304 '''
305 fp = self.fp
305 fp = self.fp
306 startTime = self.startTime
306 startTime = self.startTime
307 endTime = self.endTime
307 endTime = self.endTime
308
308
309 grp = fp['Data']
309 grp = fp['Data']
310 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
310 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
311
311
312 #ERROOOOR
312 #ERROOOOR
313 if self.timezone == 'lt':
313 if self.timezone == 'lt':
314 thisUtcTime -= 5*3600
314 thisUtcTime -= 5*3600
315
315
316 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
316 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
317
317
318 thisDate = thisDatetime.date()
318 thisDate = thisDatetime.date()
319 thisTime = thisDatetime.time()
319 thisTime = thisDatetime.time()
320
320
321 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
321 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
322 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
322 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323
323
324 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
324 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
325
325
326 self.blockList = ind
326 self.blockList = ind
327 self.blocksPerFile = len(ind)
327 self.blocksPerFile = len(ind)
328
328
329 return
329 return
330
330
331 def __readMetadata(self):
331 def __readMetadata(self):
332 '''
332 '''
333 Reads Metadata
333 Reads Metadata
334
334
335 self.pathMeta
335 self.pathMeta
336
336
337 self.listShapes
337 self.listShapes
338 self.listMetaname
338 self.listMetaname
339 self.listMeta
339 self.listMeta
340
340
341 '''
341 '''
342
342
343 # grp = self.fp['Data']
343 # grp = self.fp['Data']
344 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
344 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
345 #
345 #
346 # if pathMeta == self.pathMeta:
346 # if pathMeta == self.pathMeta:
347 # return
347 # return
348 # else:
348 # else:
349 # self.pathMeta = pathMeta
349 # self.pathMeta = pathMeta
350 #
350 #
351 # filePointer = h5py.File(self.pathMeta,'r')
351 # filePointer = h5py.File(self.pathMeta,'r')
352 # groupPointer = filePointer['Metadata']
352 # groupPointer = filePointer['Metadata']
353
353
354 filename = self.filenameList[0]
354 filename = self.filenameList[0]
355
355
356 fp = h5py.File(filename,'r')
356 fp = h5py.File(filename,'r')
357
357
358 gp = fp['Metadata']
358 gp = fp['Metadata']
359
359
360 listMetaname = []
360 listMetaname = []
361 listMetadata = []
361 listMetadata = []
362 for item in list(gp.items()):
362 for item in list(gp.items()):
363 name = item[0]
363 name = item[0]
364
364
365 if name=='array dimensions':
365 if name=='array dimensions':
366 table = gp[name][:]
366 table = gp[name][:]
367 listShapes = {}
367 listShapes = {}
368 for shapes in table:
368 for shapes in table:
369 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
369 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
370 else:
370 else:
371 data = gp[name].value
371 data = gp[name].value
372 listMetaname.append(name)
372 listMetaname.append(name)
373 listMetadata.append(data)
373 listMetadata.append(data)
374
374
375 # if name=='type':
375 # if name=='type':
376 # self.__initDataOut(data)
376 # self.__initDataOut(data)
377
377
378 self.listShapes = listShapes
378 self.listShapes = listShapes
379 self.listMetaname = listMetaname
379 self.listMetaname = listMetaname
380 self.listMeta = listMetadata
380 self.listMeta = listMetadata
381
381
382 fp.close()
382 fp.close()
383 return
383 return
384
384
385 def __readData(self):
385 def __readData(self):
386 grp = self.fp['Data']
386 grp = self.fp['Data']
387 listdataname = []
387 listdataname = []
388 listdata = []
388 listdata = []
389
389
390 for item in list(grp.items()):
390 for item in list(grp.items()):
391 name = item[0]
391 name = item[0]
392 listdataname.append(name)
392 listdataname.append(name)
393
393
394 array = self.__setDataArray(grp[name],self.listShapes[name])
394 array = self.__setDataArray(grp[name],self.listShapes[name])
395 listdata.append(array)
395 listdata.append(array)
396
396
397 self.listDataname = listdataname
397 self.listDataname = listdataname
398 self.listData = listdata
398 self.listData = listdata
399 return
399 return
400
400
401 def __setDataArray(self, dataset, shapes):
401 def __setDataArray(self, dataset, shapes):
402
402
403 nDims = shapes[0]
403 nDims = shapes[0]
404
404
405 nDim2 = shapes[1] #Dimension 0
405 nDim2 = shapes[1] #Dimension 0
406
406
407 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
407 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
408
408
409 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
409 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
410
410
411 mode = shapes[4] #Mode of storing
411 mode = shapes[4] #Mode of storing
412
412
413 blockList = self.blockList
413 blockList = self.blockList
414
414
415 blocksPerFile = self.blocksPerFile
415 blocksPerFile = self.blocksPerFile
416
416
417 #Depending on what mode the data was stored
417 #Depending on what mode the data was stored
418 if mode == 0: #Divided in channels
418 if mode == 0: #Divided in channels
419 arrayData = dataset.value.astype(numpy.float)[0][blockList]
419 arrayData = dataset.value.astype(numpy.float)[0][blockList]
420 if mode == 1: #Divided in parameter
420 if mode == 1: #Divided in parameter
421 strds = 'table'
421 strds = 'table'
422 nDatas = nDim1
422 nDatas = nDim1
423 newShapes = (blocksPerFile,nDim2,nDim0)
423 newShapes = (blocksPerFile,nDim2,nDim0)
424 elif mode==2: #Concatenated in a table
424 elif mode==2: #Concatenated in a table
425 strds = 'table0'
425 strds = 'table0'
426 arrayData = dataset[strds].value
426 arrayData = dataset[strds].value
427 #Selecting part of the dataset
427 #Selecting part of the dataset
428 utctime = arrayData[:,0]
428 utctime = arrayData[:,0]
429 u, indices = numpy.unique(utctime, return_index=True)
429 u, indices = numpy.unique(utctime, return_index=True)
430
430
431 if blockList.size != indices.size:
431 if blockList.size != indices.size:
432 indMin = indices[blockList[0]]
432 indMin = indices[blockList[0]]
433 if blockList[1] + 1 >= indices.size:
433 if blockList[1] + 1 >= indices.size:
434 arrayData = arrayData[indMin:,:]
434 arrayData = arrayData[indMin:,:]
435 else:
435 else:
436 indMax = indices[blockList[1] + 1]
436 indMax = indices[blockList[1] + 1]
437 arrayData = arrayData[indMin:indMax,:]
437 arrayData = arrayData[indMin:indMax,:]
438 return arrayData
438 return arrayData
439
439
440 # One dimension
440 # One dimension
441 if nDims == 0:
441 if nDims == 0:
442 arrayData = dataset.value.astype(numpy.float)[0][blockList]
442 arrayData = dataset.value.astype(numpy.float)[0][blockList]
443
443
444 # Two dimensions
444 # Two dimensions
445 elif nDims == 2:
445 elif nDims == 2:
446 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
446 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
447 newShapes = (blocksPerFile,nDim0)
447 newShapes = (blocksPerFile,nDim0)
448 nDatas = nDim1
448 nDatas = nDim1
449
449
450 for i in range(nDatas):
450 for i in range(nDatas):
451 data = dataset[strds + str(i)].value
451 data = dataset[strds + str(i)].value
452 arrayData[:,i,:] = data[blockList,:]
452 arrayData[:,i,:] = data[blockList,:]
453
453
454 # Three dimensions
454 # Three dimensions
455 else:
455 else:
456 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
456 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
457 for i in range(nDatas):
457 for i in range(nDatas):
458
458
459 data = dataset[strds + str(i)].value
459 data = dataset[strds + str(i)].value
460
460
461 for b in range(blockList.size):
461 for b in range(blockList.size):
462 arrayData[b,:,i,:] = data[:,:,blockList[b]]
462 arrayData[b,:,i,:] = data[:,:,blockList[b]]
463
463
464 return arrayData
464 return arrayData
465
465
466 def __setDataOut(self):
466 def __setDataOut(self):
467 listMeta = self.listMeta
467 listMeta = self.listMeta
468 listMetaname = self.listMetaname
468 listMetaname = self.listMetaname
469 listDataname = self.listDataname
469 listDataname = self.listDataname
470 listData = self.listData
470 listData = self.listData
471 listShapes = self.listShapes
471 listShapes = self.listShapes
472
472
473 blockIndex = self.blockIndex
473 blockIndex = self.blockIndex
474 # blockList = self.blockList
474 # blockList = self.blockList
475
475
476 for i in range(len(listMeta)):
476 for i in range(len(listMeta)):
477 setattr(self.dataOut,listMetaname[i],listMeta[i])
477 setattr(self.dataOut,listMetaname[i],listMeta[i])
478
478
479 for j in range(len(listData)):
479 for j in range(len(listData)):
480 nShapes = listShapes[listDataname[j]][0]
480 nShapes = listShapes[listDataname[j]][0]
481 mode = listShapes[listDataname[j]][4]
481 mode = listShapes[listDataname[j]][4]
482 if nShapes == 1:
482 if nShapes == 1:
483 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
483 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
484 elif nShapes > 1:
484 elif nShapes > 1:
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
486 elif mode==0:
486 elif mode==0:
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
488 #Mode Meteors
488 #Mode Meteors
489 elif mode ==2:
489 elif mode ==2:
490 selectedData = self.__selectDataMode2(listData[j], blockIndex)
490 selectedData = self.__selectDataMode2(listData[j], blockIndex)
491 setattr(self.dataOut, listDataname[j], selectedData)
491 setattr(self.dataOut, listDataname[j], selectedData)
492 return
492 return
493
493
494 def __selectDataMode2(self, data, blockIndex):
494 def __selectDataMode2(self, data, blockIndex):
495 utctime = data[:,0]
495 utctime = data[:,0]
496 aux, indices = numpy.unique(utctime, return_inverse=True)
496 aux, indices = numpy.unique(utctime, return_inverse=True)
497 selInd = numpy.where(indices == blockIndex)[0]
497 selInd = numpy.where(indices == blockIndex)[0]
498 selData = data[selInd,:]
498 selData = data[selInd,:]
499
499
500 return selData
500 return selData
501
501
502 def getData(self):
502 def getData(self):
503
503
504 if self.blockIndex==self.blocksPerFile:
504 if self.blockIndex==self.blocksPerFile:
505 if not( self.__setNextFileOffline() ):
505 if not( self.__setNextFileOffline() ):
506 self.dataOut.flagNoData = True
506 self.dataOut.flagNoData = True
507 return 0
507 return 0
508
508
509 self.__setDataOut()
509 self.__setDataOut()
510 self.dataOut.flagNoData = False
510 self.dataOut.flagNoData = False
511
511
512 self.blockIndex += 1
512 self.blockIndex += 1
513
513
514 return
514 return
515
515
516 def run(self, **kwargs):
516 def run(self, **kwargs):
517
517
518 if not(self.isConfig):
518 if not(self.isConfig):
519 self.setup(**kwargs)
519 self.setup(**kwargs)
520 # self.setObjProperties()
520 # self.setObjProperties()
521 self.isConfig = True
521 self.isConfig = True
522
522
523 self.getData()
523 self.getData()
524
524
525 return
525 return
526
526
527 @MPDecorator
527 @MPDecorator
528 class ParamWriter(Operation):
528 class ParamWriter(Operation):
529 '''
529 '''
530 HDF5 Writer, stores parameters data in HDF5 format files
530 HDF5 Writer, stores parameters data in HDF5 format files
531
531
532 path: path where the files will be stored
532 path: path where the files will be stored
533
533
534 blocksPerFile: number of blocks that will be saved in per HDF5 format file
534 blocksPerFile: number of blocks that will be saved in per HDF5 format file
535
535
536 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
536 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
537
537
538 metadataList: list of attributes that will be stored as metadata
538 metadataList: list of attributes that will be stored as metadata
539
539
540 dataList: list of attributes that will be stores as data
540 dataList: list of attributes that will be stores as data
541
541
542 '''
542 '''
543
543
544
544
545 ext = ".hdf5"
545 ext = ".hdf5"
546 optchar = "D"
546 optchar = "D"
547 metaoptchar = "M"
547 metaoptchar = "M"
548 metaFile = None
548 metaFile = None
549 filename = None
549 filename = None
550 path = None
550 path = None
551 setFile = None
551 setFile = None
552 fp = None
552 fp = None
553 grp = None
553 grp = None
554 ds = None
554 ds = None
555 firsttime = True
555 firsttime = True
556 #Configurations
556 #Configurations
557 blocksPerFile = None
557 blocksPerFile = None
558 blockIndex = None
558 blockIndex = None
559 dataOut = None
559 dataOut = None
560 #Data Arrays
560 #Data Arrays
561 dataList = None
561 dataList = None
562 metadataList = None
562 metadataList = None
563 dsList = None #List of dictionaries with dataset properties
563 dsList = None #List of dictionaries with dataset properties
564 tableDim = None
564 tableDim = None
565 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
565 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
566 currentDay = None
566 currentDay = None
567 lastTime = None
567 lastTime = None
568
568
569 def __init__(self):
569 def __init__(self):
570
570
571 Operation.__init__(self)
571 Operation.__init__(self)
572 return
572 return
573
573
574 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
574 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
575 self.path = path
575 self.path = path
576 self.blocksPerFile = blocksPerFile
576 self.blocksPerFile = blocksPerFile
577 self.metadataList = metadataList
577 self.metadataList = metadataList
578 self.dataList = dataList
578 self.dataList = dataList
579 self.dataOut = dataOut
579 self.dataOut = dataOut
580 self.mode = mode
580 self.mode = mode
581 if self.mode is not None:
581 if self.mode is not None:
582 self.mode = numpy.zeros(len(self.dataList)) + mode
582 self.mode = numpy.zeros(len(self.dataList)) + mode
583 else:
583 else:
584 self.mode = numpy.ones(len(self.dataList))
584 self.mode = numpy.ones(len(self.dataList))
585
585
586 self.setType = setType
586 self.setType = setType
587
587
588 arrayDim = numpy.zeros((len(self.dataList),5))
588 arrayDim = numpy.zeros((len(self.dataList),5))
589
589
590 #Table dimensions
590 #Table dimensions
591 dtype0 = self.dtype
591 dtype0 = self.dtype
592 tableList = []
592 tableList = []
593
593
594 #Dictionary and list of tables
594 #Dictionary and list of tables
595 dsList = []
595 dsList = []
596
596
597 for i in range(len(self.dataList)):
597 for i in range(len(self.dataList)):
598 dsDict = {}
598 dsDict = {}
599 dataAux = getattr(self.dataOut, self.dataList[i])
599 dataAux = getattr(self.dataOut, self.dataList[i])
600 dsDict['variable'] = self.dataList[i]
600 dsDict['variable'] = self.dataList[i]
601 #--------------------- Conditionals ------------------------
601 #--------------------- Conditionals ------------------------
602 #There is no data
602 #There is no data
603
603
604
605 if dataAux is None:
604 if dataAux is None:
606
605
607 return 0
606 return 0
608
607
609 if type(dataAux)==float or type(dataAux)==int:
608 if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
610 dsDict['mode'] = 0
609 dsDict['mode'] = 0
611 dsDict['nDim'] = 0
610 dsDict['nDim'] = 0
612 arrayDim[i,0] = 0
611 arrayDim[i,0] = 0
613 dsList.append(dsDict)
612 dsList.append(dsDict)
614
613
615 #Mode 2: meteors
614 #Mode 2: meteors
616 elif self.mode[i] == 2:
615 elif self.mode[i] == 2:
617 dsDict['dsName'] = 'table0'
616 dsDict['dsName'] = 'table0'
618 dsDict['mode'] = 2 # Mode meteors
617 dsDict['mode'] = 2 # Mode meteors
619 dsDict['shape'] = dataAux.shape[-1]
618 dsDict['shape'] = dataAux.shape[-1]
620 dsDict['nDim'] = 0
619 dsDict['nDim'] = 0
621 dsDict['dsNumber'] = 1
620 dsDict['dsNumber'] = 1
622 arrayDim[i,3] = dataAux.shape[-1]
621 arrayDim[i,3] = dataAux.shape[-1]
623 arrayDim[i,4] = self.mode[i] #Mode the data was stored
622 arrayDim[i,4] = self.mode[i] #Mode the data was stored
624 dsList.append(dsDict)
623 dsList.append(dsDict)
625
624
626 #Mode 1
625 #Mode 1
627 else:
626 else:
628 arrayDim0 = dataAux.shape #Data dimensions
627 arrayDim0 = dataAux.shape #Data dimensions
629 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
628 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
630 arrayDim[i,4] = self.mode[i] #Mode the data was stored
629 arrayDim[i,4] = self.mode[i] #Mode the data was stored
631 strtable = 'table'
630 strtable = 'table'
632 dsDict['mode'] = 1 # Mode parameters
631 dsDict['mode'] = 1 # Mode parameters
633
632
634 # Three-dimension arrays
633 # Three-dimension arrays
635 if len(arrayDim0) == 3:
634 if len(arrayDim0) == 3:
636 arrayDim[i,1:-1] = numpy.array(arrayDim0)
635 arrayDim[i,1:-1] = numpy.array(arrayDim0)
637 nTables = int(arrayDim[i,2])
636 nTables = int(arrayDim[i,2])
638 dsDict['dsNumber'] = nTables
637 dsDict['dsNumber'] = nTables
639 dsDict['shape'] = arrayDim[i,2:4]
638 dsDict['shape'] = arrayDim[i,2:4]
640 dsDict['nDim'] = 3
639 dsDict['nDim'] = 3
641
640
642 for j in range(nTables):
641 for j in range(nTables):
643 dsDict = dsDict.copy()
642 dsDict = dsDict.copy()
644 dsDict['dsName'] = strtable + str(j)
643 dsDict['dsName'] = strtable + str(j)
645 dsList.append(dsDict)
644 dsList.append(dsDict)
646
645
647 # Two-dimension arrays
646 # Two-dimension arrays
648 elif len(arrayDim0) == 2:
647 elif len(arrayDim0) == 2:
649 arrayDim[i,2:-1] = numpy.array(arrayDim0)
648 arrayDim[i,2:-1] = numpy.array(arrayDim0)
650 nTables = int(arrayDim[i,2])
649 nTables = int(arrayDim[i,2])
651 dsDict['dsNumber'] = nTables
650 dsDict['dsNumber'] = nTables
652 dsDict['shape'] = arrayDim[i,3]
651 dsDict['shape'] = arrayDim[i,3]
653 dsDict['nDim'] = 2
652 dsDict['nDim'] = 2
654
653
655 for j in range(nTables):
654 for j in range(nTables):
656 dsDict = dsDict.copy()
655 dsDict = dsDict.copy()
657 dsDict['dsName'] = strtable + str(j)
656 dsDict['dsName'] = strtable + str(j)
658 dsList.append(dsDict)
657 dsList.append(dsDict)
659
658
660 # One-dimension arrays
659 # One-dimension arrays
661 elif len(arrayDim0) == 1:
660 elif len(arrayDim0) == 1:
662 arrayDim[i,3] = arrayDim0[0]
661 arrayDim[i,3] = arrayDim0[0]
663 dsDict['shape'] = arrayDim0[0]
662 dsDict['shape'] = arrayDim0[0]
664 dsDict['dsNumber'] = 1
663 dsDict['dsNumber'] = 1
665 dsDict['dsName'] = strtable + str(0)
664 dsDict['dsName'] = strtable + str(0)
666 dsDict['nDim'] = 1
665 dsDict['nDim'] = 1
667 dsList.append(dsDict)
666 dsList.append(dsDict)
668
667
669 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
668 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
670 tableList.append(table)
669 tableList.append(table)
671
670
672 self.dsList = dsList
671 self.dsList = dsList
673 self.tableDim = numpy.array(tableList, dtype = dtype0)
672 self.tableDim = numpy.array(tableList, dtype = dtype0)
674 self.blockIndex = 0
673 self.blockIndex = 0
675 timeTuple = time.localtime(dataOut.utctime)
674 timeTuple = time.localtime(dataOut.utctime)
676 self.currentDay = timeTuple.tm_yday
675 self.currentDay = timeTuple.tm_yday
677
676
678 def putMetadata(self):
677 def putMetadata(self):
679
678
680 fp = self.createMetadataFile()
679 fp = self.createMetadataFile()
681 self.writeMetadata(fp)
680 self.writeMetadata(fp)
682 fp.close()
681 fp.close()
683 return
682 return
684
683
685 def createMetadataFile(self):
684 def createMetadataFile(self):
686 ext = self.ext
685 ext = self.ext
687 path = self.path
686 path = self.path
688 setFile = self.setFile
687 setFile = self.setFile
689
688
690 timeTuple = time.localtime(self.dataOut.utctime)
689 timeTuple = time.localtime(self.dataOut.utctime)
691
690
692 subfolder = ''
691 subfolder = ''
693 fullpath = os.path.join( path, subfolder )
692 fullpath = os.path.join( path, subfolder )
694
693
695 if not( os.path.exists(fullpath) ):
694 if not( os.path.exists(fullpath) ):
696 os.mkdir(fullpath)
695 os.mkdir(fullpath)
697 setFile = -1 #inicializo mi contador de seteo
696 setFile = -1 #inicializo mi contador de seteo
698
697
699 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
698 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
700 fullpath = os.path.join( path, subfolder )
699 fullpath = os.path.join( path, subfolder )
701
700
702 if not( os.path.exists(fullpath) ):
701 if not( os.path.exists(fullpath) ):
703 os.mkdir(fullpath)
702 os.mkdir(fullpath)
704 setFile = -1 #inicializo mi contador de seteo
703 setFile = -1 #inicializo mi contador de seteo
705
704
706 else:
705 else:
707 filesList = os.listdir( fullpath )
706 filesList = os.listdir( fullpath )
708 filesList = sorted( filesList, key=str.lower )
707 filesList = sorted( filesList, key=str.lower )
709 if len( filesList ) > 0:
708 if len( filesList ) > 0:
710 filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
709 filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
711 filen = filesList[-1]
710 filen = filesList[-1]
712 # el filename debera tener el siguiente formato
711 # el filename debera tener el siguiente formato
713 # 0 1234 567 89A BCDE (hex)
712 # 0 1234 567 89A BCDE (hex)
714 # x YYYY DDD SSS .ext
713 # x YYYY DDD SSS .ext
715 if isNumber( filen[8:11] ):
714 if isNumber( filen[8:11] ):
716 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
715 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
717 else:
716 else:
718 setFile = -1
717 setFile = -1
719 else:
718 else:
720 setFile = -1 #inicializo mi contador de seteo
719 setFile = -1 #inicializo mi contador de seteo
721
720
722 if self.setType is None:
721 if self.setType is None:
723 setFile += 1
722 setFile += 1
724 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
723 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
725 timeTuple.tm_year,
724 timeTuple.tm_year,
726 timeTuple.tm_yday,
725 timeTuple.tm_yday,
727 setFile,
726 setFile,
728 ext )
727 ext )
729 else:
728 else:
730 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
729 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
731 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
730 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
732 timeTuple.tm_year,
731 timeTuple.tm_year,
733 timeTuple.tm_yday,
732 timeTuple.tm_yday,
734 setFile,
733 setFile,
735 ext )
734 ext )
736
735
737 filename = os.path.join( path, subfolder, file )
736 filename = os.path.join( path, subfolder, file )
738 self.metaFile = file
737 self.metaFile = file
739 #Setting HDF5 File
738 #Setting HDF5 File
740 fp = h5py.File(filename,'w')
739 fp = h5py.File(filename,'w')
741
740
742 return fp
741 return fp
743
742
744 def writeMetadata(self, fp):
743 def writeMetadata(self, fp):
745
744
746 grp = fp.create_group("Metadata")
745 grp = fp.create_group("Metadata")
747 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
746 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
748
747
749 for i in range(len(self.metadataList)):
748 for i in range(len(self.metadataList)):
750 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
749 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
751 return
750 return
752
751
753 def timeFlag(self):
752 def timeFlag(self):
754 currentTime = self.dataOut.utctime
753 currentTime = self.dataOut.utctime
755
754
756 if self.lastTime is None:
755 if self.lastTime is None:
757 self.lastTime = currentTime
756 self.lastTime = currentTime
758
757
759 #Day
758 #Day
760 timeTuple = time.localtime(currentTime)
759 timeTuple = time.localtime(currentTime)
761 dataDay = timeTuple.tm_yday
760 dataDay = timeTuple.tm_yday
762
761
763 #Time
762 #Time
764 timeDiff = currentTime - self.lastTime
763 timeDiff = currentTime - self.lastTime
765
764
766 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
765 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
767 if dataDay != self.currentDay:
766 if dataDay != self.currentDay:
768 self.currentDay = dataDay
767 self.currentDay = dataDay
769 return True
768 return True
770 elif timeDiff > 3*60*60:
769 elif timeDiff > 3*60*60:
771 self.lastTime = currentTime
770 self.lastTime = currentTime
772 return True
771 return True
773 else:
772 else:
774 self.lastTime = currentTime
773 self.lastTime = currentTime
775 return False
774 return False
776
775
777 def setNextFile(self):
776 def setNextFile(self):
778
777
779 ext = self.ext
778 ext = self.ext
780 path = self.path
779 path = self.path
781 setFile = self.setFile
780 setFile = self.setFile
782 mode = self.mode
781 mode = self.mode
783
782
784 timeTuple = time.localtime(self.dataOut.utctime)
783 timeTuple = time.localtime(self.dataOut.utctime)
785 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
784 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
786
785
787 fullpath = os.path.join( path, subfolder )
786 fullpath = os.path.join( path, subfolder )
788
787
789 if os.path.exists(fullpath):
788 if os.path.exists(fullpath):
790 filesList = os.listdir( fullpath )
789 filesList = os.listdir( fullpath )
791 filesList = [k for k in filesList if k.startswith(self.optchar)]
790 filesList = [k for k in filesList if k.startswith(self.optchar)]
792 if len( filesList ) > 0:
791 if len( filesList ) > 0:
793 filesList = sorted( filesList, key=str.lower )
792 filesList = sorted( filesList, key=str.lower )
794 filen = filesList[-1]
793 filen = filesList[-1]
795 # el filename debera tener el siguiente formato
794 # el filename debera tener el siguiente formato
796 # 0 1234 567 89A BCDE (hex)
795 # 0 1234 567 89A BCDE (hex)
797 # x YYYY DDD SSS .ext
796 # x YYYY DDD SSS .ext
798 if isNumber( filen[8:11] ):
797 if isNumber( filen[8:11] ):
799 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
798 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
800 else:
799 else:
801 setFile = -1
800 setFile = -1
802 else:
801 else:
803 setFile = -1 #inicializo mi contador de seteo
802 setFile = -1 #inicializo mi contador de seteo
804 else:
803 else:
805 os.makedirs(fullpath)
804 os.makedirs(fullpath)
806 setFile = -1 #inicializo mi contador de seteo
805 setFile = -1 #inicializo mi contador de seteo
807
806
808 if self.setType is None:
807 if self.setType is None:
809 setFile += 1
808 setFile += 1
810 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
809 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
811 timeTuple.tm_year,
810 timeTuple.tm_year,
812 timeTuple.tm_yday,
811 timeTuple.tm_yday,
813 setFile,
812 setFile,
814 ext )
813 ext )
815 else:
814 else:
816 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
815 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
817 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
816 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
818 timeTuple.tm_year,
817 timeTuple.tm_year,
819 timeTuple.tm_yday,
818 timeTuple.tm_yday,
820 setFile,
819 setFile,
821 ext )
820 ext )
822
821
823 filename = os.path.join( path, subfolder, file )
822 filename = os.path.join( path, subfolder, file )
824
823
825 #Setting HDF5 File
824 #Setting HDF5 File
826 fp = h5py.File(filename,'w')
825 fp = h5py.File(filename,'w')
827 #write metadata
826 #write metadata
828 self.writeMetadata(fp)
827 self.writeMetadata(fp)
829 #Write data
828 #Write data
830 grp = fp.create_group("Data")
829 grp = fp.create_group("Data")
831 ds = []
830 ds = []
832 data = []
831 data = []
833 dsList = self.dsList
832 dsList = self.dsList
834 i = 0
833 i = 0
835 while i < len(dsList):
834 while i < len(dsList):
836 dsInfo = dsList[i]
835 dsInfo = dsList[i]
837 #One-dimension data
836 #One-dimension data
838 if dsInfo['mode'] == 0:
837 if dsInfo['mode'] == 0:
839 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
838 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
840 ds.append(ds0)
839 ds.append(ds0)
841 data.append([])
840 data.append([])
842 i += 1
841 i += 1
843 continue
842 continue
844
843
845 elif dsInfo['mode'] == 2:
844 elif dsInfo['mode'] == 2:
846 grp0 = grp.create_group(dsInfo['variable'])
845 grp0 = grp.create_group(dsInfo['variable'])
847 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
846 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
848 ds.append(ds0)
847 ds.append(ds0)
849 data.append([])
848 data.append([])
850 i += 1
849 i += 1
851 continue
850 continue
852
851
853 elif dsInfo['mode'] == 1:
852 elif dsInfo['mode'] == 1:
854 grp0 = grp.create_group(dsInfo['variable'])
853 grp0 = grp.create_group(dsInfo['variable'])
855
854
856 for j in range(dsInfo['dsNumber']):
855 for j in range(dsInfo['dsNumber']):
857 dsInfo = dsList[i]
856 dsInfo = dsList[i]
858 tableName = dsInfo['dsName']
857 tableName = dsInfo['dsName']
859
858
860
859
861 if dsInfo['nDim'] == 3:
860 if dsInfo['nDim'] == 3:
862 shape = dsInfo['shape'].astype(int)
861 shape = dsInfo['shape'].astype(int)
863 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
862 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
864 else:
863 else:
865 shape = int(dsInfo['shape'])
864 shape = int(dsInfo['shape'])
866 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
865 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
867
866
868 ds.append(ds0)
867 ds.append(ds0)
869 data.append([])
868 data.append([])
870 i += 1
869 i += 1
871
870
872 fp.flush()
871 fp.flush()
873 fp.close()
872 fp.close()
874
873
875 log.log('creating file: {}'.format(filename), 'Writing')
874 log.log('creating file: {}'.format(filename), 'Writing')
876 self.filename = filename
875 self.filename = filename
877 self.ds = ds
876 self.ds = ds
878 self.data = data
877 self.data = data
879 self.firsttime = True
878 self.firsttime = True
880 self.blockIndex = 0
879 self.blockIndex = 0
881 return
880 return
882
881
883 def putData(self):
882 def putData(self):
884
883
885 if self.blockIndex == self.blocksPerFile or self.timeFlag():
884 if self.blockIndex == self.blocksPerFile or self.timeFlag():
886 self.setNextFile()
885 self.setNextFile()
887
886
888 self.readBlock()
887 self.readBlock()
889 self.setBlock() #Prepare data to be written
888 self.setBlock() #Prepare data to be written
890 self.writeBlock() #Write data
889 self.writeBlock() #Write data
891
890
892 return
891 return
893
892
894 def readBlock(self):
893 def readBlock(self):
895
894
896 '''
895 '''
897 data Array configured
896 data Array configured
898
897
899
898
900 self.data
899 self.data
901 '''
900 '''
902 dsList = self.dsList
901 dsList = self.dsList
903 ds = self.ds
902 ds = self.ds
904 #Setting HDF5 File
903 #Setting HDF5 File
905 fp = h5py.File(self.filename,'r+')
904 fp = h5py.File(self.filename,'r+')
906 grp = fp["Data"]
905 grp = fp["Data"]
907 ind = 0
906 ind = 0
908
907
909 while ind < len(dsList):
908 while ind < len(dsList):
910 dsInfo = dsList[ind]
909 dsInfo = dsList[ind]
911
910
912 if dsInfo['mode'] == 0:
911 if dsInfo['mode'] == 0:
913 ds0 = grp[dsInfo['variable']]
912 ds0 = grp[dsInfo['variable']]
914 ds[ind] = ds0
913 ds[ind] = ds0
915 ind += 1
914 ind += 1
916 else:
915 else:
917
916
918 grp0 = grp[dsInfo['variable']]
917 grp0 = grp[dsInfo['variable']]
919
918
920 for j in range(dsInfo['dsNumber']):
919 for j in range(dsInfo['dsNumber']):
921 dsInfo = dsList[ind]
920 dsInfo = dsList[ind]
922 ds0 = grp0[dsInfo['dsName']]
921 ds0 = grp0[dsInfo['dsName']]
923 ds[ind] = ds0
922 ds[ind] = ds0
924 ind += 1
923 ind += 1
925
924
926 self.fp = fp
925 self.fp = fp
927 self.grp = grp
926 self.grp = grp
928 self.ds = ds
927 self.ds = ds
929
928
930 return
929 return
931
930
932 def setBlock(self):
931 def setBlock(self):
933 '''
932 '''
934 data Array configured
933 data Array configured
935
934
936
935
937 self.data
936 self.data
938 '''
937 '''
939 #Creating Arrays
938 #Creating Arrays
940 dsList = self.dsList
939 dsList = self.dsList
941 data = self.data
940 data = self.data
942 ind = 0
941 ind = 0
943
942
944 while ind < len(dsList):
943 while ind < len(dsList):
945 dsInfo = dsList[ind]
944 dsInfo = dsList[ind]
946 dataAux = getattr(self.dataOut, dsInfo['variable'])
945 dataAux = getattr(self.dataOut, dsInfo['variable'])
947
946
948 mode = dsInfo['mode']
947 mode = dsInfo['mode']
949 nDim = dsInfo['nDim']
948 nDim = dsInfo['nDim']
950
949
951 if mode == 0 or mode == 2 or nDim == 1:
950 if mode == 0 or mode == 2 or nDim == 1:
952 data[ind] = dataAux
951 data[ind] = dataAux
953 ind += 1
952 ind += 1
954 # elif nDim == 1:
953 # elif nDim == 1:
955 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
954 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
956 # ind += 1
955 # ind += 1
957 elif nDim == 2:
956 elif nDim == 2:
958 for j in range(dsInfo['dsNumber']):
957 for j in range(dsInfo['dsNumber']):
959 data[ind] = dataAux[j,:]
958 data[ind] = dataAux[j,:]
960 ind += 1
959 ind += 1
961 elif nDim == 3:
960 elif nDim == 3:
962 for j in range(dsInfo['dsNumber']):
961 for j in range(dsInfo['dsNumber']):
963 data[ind] = dataAux[:,j,:]
962 data[ind] = dataAux[:,j,:]
964 ind += 1
963 ind += 1
965
964
966 self.data = data
965 self.data = data
967 return
966 return
968
967
969 def writeBlock(self):
968 def writeBlock(self):
970 '''
969 '''
971 Saves the block in the HDF5 file
970 Saves the block in the HDF5 file
972 '''
971 '''
973 dsList = self.dsList
972 dsList = self.dsList
974
973
975 for i in range(len(self.ds)):
974 for i in range(len(self.ds)):
976 dsInfo = dsList[i]
975 dsInfo = dsList[i]
977 nDim = dsInfo['nDim']
976 nDim = dsInfo['nDim']
978 mode = dsInfo['mode']
977 mode = dsInfo['mode']
979
978
980 # First time
979 # First time
981 if self.firsttime:
980 if self.firsttime:
982 if type(self.data[i]) == numpy.ndarray:
981 if type(self.data[i]) == numpy.ndarray:
983
982
984 if nDim == 3:
983 if nDim == 3:
985 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
984 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
986 self.ds[i].resize(self.data[i].shape)
985 self.ds[i].resize(self.data[i].shape)
987 if mode == 2:
986 if mode == 2:
988 self.ds[i].resize(self.data[i].shape)
987 self.ds[i].resize(self.data[i].shape)
989 self.ds[i][:] = self.data[i]
988 self.ds[i][:] = self.data[i]
990 else:
989 else:
991
990
992 # From second time
991 # From second time
993 # Meteors!
992 # Meteors!
994 if mode == 2:
993 if mode == 2:
995 dataShape = self.data[i].shape
994 dataShape = self.data[i].shape
996 dsShape = self.ds[i].shape
995 dsShape = self.ds[i].shape
997 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
996 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
998 self.ds[i][dsShape[0]:,:] = self.data[i]
997 self.ds[i][dsShape[0]:,:] = self.data[i]
999 # No dimension
998 # No dimension
1000 elif mode == 0:
999 elif mode == 0:
1001 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1000 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1002 self.ds[i][0,-1] = self.data[i]
1001 self.ds[i][0,-1] = self.data[i]
1003 # One dimension
1002 # One dimension
1004 elif nDim == 1:
1003 elif nDim == 1:
1005 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1004 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1006 self.ds[i][-1,:] = self.data[i]
1005 self.ds[i][-1,:] = self.data[i]
1007 # Two dimension
1006 # Two dimension
1008 elif nDim == 2:
1007 elif nDim == 2:
1009 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1008 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1010 self.ds[i][self.blockIndex,:] = self.data[i]
1009 self.ds[i][self.blockIndex,:] = self.data[i]
1011 # Three dimensions
1010 # Three dimensions
1012 elif nDim == 3:
1011 elif nDim == 3:
1013 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1012 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1014 self.ds[i][:,:,-1] = self.data[i]
1013 self.ds[i][:,:,-1] = self.data[i]
1015
1014
1016 self.firsttime = False
1015 self.firsttime = False
1017 self.blockIndex += 1
1016 self.blockIndex += 1
1018
1017
1019 #Close to save changes
1018 #Close to save changes
1020 self.fp.flush()
1019 self.fp.flush()
1021 self.fp.close()
1020 self.fp.close()
1022 return
1021 return
1023
1022
1024 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
1023 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
1025
1024
1026 if not(self.isConfig):
1025 if not(self.isConfig):
1027 self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1026 self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1028 metadataList=metadataList, dataList=dataList, mode=mode,
1027 metadataList=metadataList, dataList=dataList, mode=mode,
1029 setType=setType)
1028 setType=setType)
1030
1029
1031 self.isConfig = True
1030 self.isConfig = True
1032 self.setNextFile()
1031 self.setNextFile()
1033
1032
1034 self.putData()
1033 self.putData()
1035 return
1034 return
1036 No newline at end of file
1035
General Comments 0
You need to be logged in to leave comments. Login now