##// END OF EJS Templates
schain
RSS
Modifications in Param Writer, files now split in two when there is a big difference in time between contiguous data
Julio Valdez -
r853:3ad9b63d79aa
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1077 +1,1092
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class ParamReader(ProcessingUnit):
15 class ParamReader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self):
77 def __init__(self):
78 self.dataOut = Parameters()
78 self.dataOut = Parameters()
79 return
79 return
80
80
81 def setup(self, **kwargs):
81 def setup(self, **kwargs):
82
82
83 path = kwargs['path']
83 path = kwargs['path']
84 startDate = kwargs['startDate']
84 startDate = kwargs['startDate']
85 endDate = kwargs['endDate']
85 endDate = kwargs['endDate']
86 startTime = kwargs['startTime']
86 startTime = kwargs['startTime']
87 endTime = kwargs['endTime']
87 endTime = kwargs['endTime']
88 walk = kwargs['walk']
88 walk = kwargs['walk']
89 if kwargs.has_key('ext'):
89 if kwargs.has_key('ext'):
90 ext = kwargs['ext']
90 ext = kwargs['ext']
91 else:
91 else:
92 ext = '.hdf5'
92 ext = '.hdf5'
93 if kwargs.has_key('timezone'):
93 if kwargs.has_key('timezone'):
94 self.timezone = kwargs['timezone']
94 self.timezone = kwargs['timezone']
95 else:
95 else:
96 self.timezone = 'lt'
96 self.timezone = 'lt'
97
97
98 print "[Reading] Searching files in offline mode ..."
98 print "[Reading] Searching files in offline mode ..."
99 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
99 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 startTime=startTime, endTime=endTime,
100 startTime=startTime, endTime=endTime,
101 ext=ext, walk=walk)
101 ext=ext, walk=walk)
102
102
103 if not(filenameList):
103 if not(filenameList):
104 print "There is no files into the folder: %s"%(path)
104 print "There is no files into the folder: %s"%(path)
105 sys.exit(-1)
105 sys.exit(-1)
106
106
107 self.fileIndex = -1
107 self.fileIndex = -1
108 self.startTime = startTime
108 self.startTime = startTime
109 self.endTime = endTime
109 self.endTime = endTime
110
110
111 self.__readMetadata()
111 self.__readMetadata()
112
112
113 self.__setNextFileOffline()
113 self.__setNextFileOffline()
114
114
115 return
115 return
116
116
117 def __searchFilesOffLine(self,
117 def __searchFilesOffLine(self,
118 path,
118 path,
119 startDate=None,
119 startDate=None,
120 endDate=None,
120 endDate=None,
121 startTime=datetime.time(0,0,0),
121 startTime=datetime.time(0,0,0),
122 endTime=datetime.time(23,59,59),
122 endTime=datetime.time(23,59,59),
123 ext='.hdf5',
123 ext='.hdf5',
124 walk=True):
124 walk=True):
125
125
126 expLabel = ''
126 expLabel = ''
127 self.filenameList = []
127 self.filenameList = []
128 self.datetimeList = []
128 self.datetimeList = []
129
129
130 pathList = []
130 pathList = []
131
131
132 JRODataObj = JRODataReader()
132 JRODataObj = JRODataReader()
133 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
133 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
134
134
135 if dateList == []:
135 if dateList == []:
136 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
136 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
137 datetime.datetime.combine(startDate,startTime).ctime(),
137 datetime.datetime.combine(startDate,startTime).ctime(),
138 datetime.datetime.combine(endDate,endTime).ctime())
138 datetime.datetime.combine(endDate,endTime).ctime())
139
139
140 return None, None
140 return None, None
141
141
142 if len(dateList) > 1:
142 if len(dateList) > 1:
143 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
143 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
144 else:
144 else:
145 print "[Reading] data was found for the date %s" %(dateList[0])
145 print "[Reading] data was found for the date %s" %(dateList[0])
146
146
147 filenameList = []
147 filenameList = []
148 datetimeList = []
148 datetimeList = []
149
149
150 #----------------------------------------------------------------------------------
150 #----------------------------------------------------------------------------------
151
151
152 for thisPath in pathList:
152 for thisPath in pathList:
153 # thisPath = pathList[pathDict[file]]
153 # thisPath = pathList[pathDict[file]]
154
154
155 fileList = glob.glob1(thisPath, "*%s" %ext)
155 fileList = glob.glob1(thisPath, "*%s" %ext)
156 fileList.sort()
156 fileList.sort()
157
157
158 for file in fileList:
158 for file in fileList:
159
159
160 filename = os.path.join(thisPath,file)
160 filename = os.path.join(thisPath,file)
161
161
162 if not isFileInDateRange(filename, startDate, endDate):
162 if not isFileInDateRange(filename, startDate, endDate):
163 continue
163 continue
164
164
165 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
165 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
166
166
167 if not(thisDatetime):
167 if not(thisDatetime):
168 continue
168 continue
169
169
170 filenameList.append(filename)
170 filenameList.append(filename)
171 datetimeList.append(thisDatetime)
171 datetimeList.append(thisDatetime)
172
172
173 if not(filenameList):
173 if not(filenameList):
174 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
174 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
175 return None, None
175 return None, None
176
176
177 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
177 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
178 print
178 print
179
179
180 for i in range(len(filenameList)):
180 for i in range(len(filenameList)):
181 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
181 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
182
182
183 self.filenameList = filenameList
183 self.filenameList = filenameList
184 self.datetimeList = datetimeList
184 self.datetimeList = datetimeList
185
185
186 return pathList, filenameList
186 return pathList, filenameList
187
187
188 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
188 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
189
189
190 """
190 """
191 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
191 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
192
192
193 Inputs:
193 Inputs:
194 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
194 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
195
195
196 startDate : fecha inicial del rango seleccionado en formato datetime.date
196 startDate : fecha inicial del rango seleccionado en formato datetime.date
197
197
198 endDate : fecha final del rango seleccionado en formato datetime.date
198 endDate : fecha final del rango seleccionado en formato datetime.date
199
199
200 startTime : tiempo inicial del rango seleccionado en formato datetime.time
200 startTime : tiempo inicial del rango seleccionado en formato datetime.time
201
201
202 endTime : tiempo final del rango seleccionado en formato datetime.time
202 endTime : tiempo final del rango seleccionado en formato datetime.time
203
203
204 Return:
204 Return:
205 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
205 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
206 fecha especificado, de lo contrario retorna False.
206 fecha especificado, de lo contrario retorna False.
207
207
208 Excepciones:
208 Excepciones:
209 Si el archivo no existe o no puede ser abierto
209 Si el archivo no existe o no puede ser abierto
210 Si la cabecera no puede ser leida.
210 Si la cabecera no puede ser leida.
211
211
212 """
212 """
213
213
214 try:
214 try:
215 fp = h5py.File(filename,'r')
215 fp = h5py.File(filename,'r')
216 grp1 = fp['Data']
216 grp1 = fp['Data']
217
217
218 except IOError:
218 except IOError:
219 traceback.print_exc()
219 traceback.print_exc()
220 raise IOError, "The file %s can't be opened" %(filename)
220 raise IOError, "The file %s can't be opened" %(filename)
221 #chino rata
221 #chino rata
222 #In case has utctime attribute
222 #In case has utctime attribute
223 grp2 = grp1['utctime']
223 grp2 = grp1['utctime']
224 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
224 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
225 thisUtcTime = grp2.value[0]
225 thisUtcTime = grp2.value[0]
226
226
227 fp.close()
227 fp.close()
228
228
229 if self.timezone == 'lt':
229 if self.timezone == 'lt':
230 thisUtcTime -= 5*3600
230 thisUtcTime -= 5*3600
231
231
232 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
232 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
233 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
233 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
234 thisDate = thisDatetime.date()
234 thisDate = thisDatetime.date()
235 thisTime = thisDatetime.time()
235 thisTime = thisDatetime.time()
236
236
237 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
237 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239
239
240 #General case
240 #General case
241 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
241 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
242 #-----------o----------------------------o-----------
242 #-----------o----------------------------o-----------
243 # startTime endTime
243 # startTime endTime
244
244
245 if endTime >= startTime:
245 if endTime >= startTime:
246 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
246 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
247 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
247 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
248 return thisDatetime
248 return thisDatetime
249 return None
249 return None
250
250
251 #If endTime < startTime then endTime belongs to the next day
251 #If endTime < startTime then endTime belongs to the next day
252 #<<<<<<<<<<<o o>>>>>>>>>>>
252 #<<<<<<<<<<<o o>>>>>>>>>>>
253 #-----------o----------------------------o-----------
253 #-----------o----------------------------o-----------
254 # endTime startTime
254 # endTime startTime
255
255
256 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
256 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
257 return None
257 return None
258
258
259 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
259 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
260 return None
260 return None
261
261
262 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
262 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
263 return None
263 return None
264
264
265 return thisDatetime
265 return thisDatetime
266
266
267 def __setNextFileOffline(self):
267 def __setNextFileOffline(self):
268
268
269 self.fileIndex += 1
269 self.fileIndex += 1
270 idFile = self.fileIndex
270 idFile = self.fileIndex
271
271
272 if not(idFile < len(self.filenameList)):
272 if not(idFile < len(self.filenameList)):
273 print "No more Files"
273 print "No more Files"
274 return 0
274 return 0
275
275
276 filename = self.filenameList[idFile]
276 filename = self.filenameList[idFile]
277
277
278 filePointer = h5py.File(filename,'r')
278 filePointer = h5py.File(filename,'r')
279
279
280 self.filename = filename
280 self.filename = filename
281
281
282 self.fp = filePointer
282 self.fp = filePointer
283
283
284 print "Setting the file: %s"%self.filename
284 print "Setting the file: %s"%self.filename
285
285
286 # self.__readMetadata()
286 # self.__readMetadata()
287 self.__setBlockList()
287 self.__setBlockList()
288 self.__readData()
288 self.__readData()
289 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
289 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
290 # self.nRecords = self.fp['Data'].attrs['nRecords']
290 # self.nRecords = self.fp['Data'].attrs['nRecords']
291 self.blockIndex = 0
291 self.blockIndex = 0
292 return 1
292 return 1
293
293
294 def __setBlockList(self):
294 def __setBlockList(self):
295 '''
295 '''
296 Selects the data within the times defined
296 Selects the data within the times defined
297
297
298 self.fp
298 self.fp
299 self.startTime
299 self.startTime
300 self.endTime
300 self.endTime
301
301
302 self.blockList
302 self.blockList
303 self.blocksPerFile
303 self.blocksPerFile
304
304
305 '''
305 '''
306 fp = self.fp
306 fp = self.fp
307 startTime = self.startTime
307 startTime = self.startTime
308 endTime = self.endTime
308 endTime = self.endTime
309
309
310 grp = fp['Data']
310 grp = fp['Data']
311 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
311 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
312
312
313 #ERROOOOR
313 #ERROOOOR
314 if self.timezone == 'lt':
314 if self.timezone == 'lt':
315 thisUtcTime -= 5*3600
315 thisUtcTime -= 5*3600
316
316
317 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
317 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
318
318
319 thisDate = thisDatetime.date()
319 thisDate = thisDatetime.date()
320 thisTime = thisDatetime.time()
320 thisTime = thisDatetime.time()
321
321
322 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
322 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324
324
325 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
325 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
326
326
327 self.blockList = ind
327 self.blockList = ind
328 self.blocksPerFile = len(ind)
328 self.blocksPerFile = len(ind)
329
329
330 return
330 return
331
331
332 def __readMetadata(self):
332 def __readMetadata(self):
333 '''
333 '''
334 Reads Metadata
334 Reads Metadata
335
335
336 self.pathMeta
336 self.pathMeta
337
337
338 self.listShapes
338 self.listShapes
339 self.listMetaname
339 self.listMetaname
340 self.listMeta
340 self.listMeta
341
341
342 '''
342 '''
343
343
344 # grp = self.fp['Data']
344 # grp = self.fp['Data']
345 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
345 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
346 #
346 #
347 # if pathMeta == self.pathMeta:
347 # if pathMeta == self.pathMeta:
348 # return
348 # return
349 # else:
349 # else:
350 # self.pathMeta = pathMeta
350 # self.pathMeta = pathMeta
351 #
351 #
352 # filePointer = h5py.File(self.pathMeta,'r')
352 # filePointer = h5py.File(self.pathMeta,'r')
353 # groupPointer = filePointer['Metadata']
353 # groupPointer = filePointer['Metadata']
354
354
355 filename = self.filenameList[0]
355 filename = self.filenameList[0]
356
356
357 fp = h5py.File(filename,'r')
357 fp = h5py.File(filename,'r')
358
358
359 gp = fp['Metadata']
359 gp = fp['Metadata']
360
360
361 listMetaname = []
361 listMetaname = []
362 listMetadata = []
362 listMetadata = []
363 for item in gp.items():
363 for item in gp.items():
364 name = item[0]
364 name = item[0]
365
365
366 if name=='array dimensions':
366 if name=='array dimensions':
367 table = gp[name][:]
367 table = gp[name][:]
368 listShapes = {}
368 listShapes = {}
369 for shapes in table:
369 for shapes in table:
370 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
370 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
371 else:
371 else:
372 data = gp[name].value
372 data = gp[name].value
373 listMetaname.append(name)
373 listMetaname.append(name)
374 listMetadata.append(data)
374 listMetadata.append(data)
375
375
376 # if name=='type':
376 # if name=='type':
377 # self.__initDataOut(data)
377 # self.__initDataOut(data)
378
378
379 self.listShapes = listShapes
379 self.listShapes = listShapes
380 self.listMetaname = listMetaname
380 self.listMetaname = listMetaname
381 self.listMeta = listMetadata
381 self.listMeta = listMetadata
382
382
383 fp.close()
383 fp.close()
384 return
384 return
385
385
386 def __readData(self):
386 def __readData(self):
387 grp = self.fp['Data']
387 grp = self.fp['Data']
388 listdataname = []
388 listdataname = []
389 listdata = []
389 listdata = []
390
390
391 for item in grp.items():
391 for item in grp.items():
392 name = item[0]
392 name = item[0]
393 listdataname.append(name)
393 listdataname.append(name)
394
394
395 array = self.__setDataArray(grp[name],self.listShapes[name])
395 array = self.__setDataArray(grp[name],self.listShapes[name])
396 listdata.append(array)
396 listdata.append(array)
397
397
398 self.listDataname = listdataname
398 self.listDataname = listdataname
399 self.listData = listdata
399 self.listData = listdata
400 return
400 return
401
401
402 def __setDataArray(self, dataset, shapes):
402 def __setDataArray(self, dataset, shapes):
403
403
404 nDims = shapes[0]
404 nDims = shapes[0]
405
405
406 nDim2 = shapes[1] #Dimension 0
406 nDim2 = shapes[1] #Dimension 0
407
407
408 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
408 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409
409
410 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
410 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411
411
412 mode = shapes[4] #Mode of storing
412 mode = shapes[4] #Mode of storing
413
413
414 blockList = self.blockList
414 blockList = self.blockList
415
415
416 blocksPerFile = self.blocksPerFile
416 blocksPerFile = self.blocksPerFile
417
417
418 #Depending on what mode the data was stored
418 #Depending on what mode the data was stored
419 if mode == 0: #Divided in channels
419 if mode == 0: #Divided in channels
420 arrayData = dataset.value.astype(numpy.float)[0][blockList]
420 arrayData = dataset.value.astype(numpy.float)[0][blockList]
421 if mode == 1: #Divided in parameter
421 if mode == 1: #Divided in parameter
422 strds = 'table'
422 strds = 'table'
423 nDatas = nDim1
423 nDatas = nDim1
424 newShapes = (blocksPerFile,nDim2,nDim0)
424 newShapes = (blocksPerFile,nDim2,nDim0)
425 elif mode==2: #Concatenated in a table
425 elif mode==2: #Concatenated in a table
426 strds = 'table0'
426 strds = 'table0'
427 arrayData = dataset[strds].value
427 arrayData = dataset[strds].value
428 #Selecting part of the dataset
428 #Selecting part of the dataset
429 utctime = arrayData[:,0]
429 utctime = arrayData[:,0]
430 u, indices = numpy.unique(utctime, return_index=True)
430 u, indices = numpy.unique(utctime, return_index=True)
431
431
432 if blockList.size != indices.size:
432 if blockList.size != indices.size:
433 indMin = indices[blockList[0]]
433 indMin = indices[blockList[0]]
434 if blockList[-1] + 1 >= indices.size:
434 if blockList[-1] + 1 >= indices.size:
435 arrayData = arrayData[indMin:,:]
435 arrayData = arrayData[indMin:,:]
436 else:
436 else:
437 indMax = indices[blockList[-1] + 1]
437 indMax = indices[blockList[-1] + 1]
438 arrayData = arrayData[indMin:indMax,:]
438 arrayData = arrayData[indMin:indMax,:]
439 return arrayData
439 return arrayData
440
440
441 #------- One dimension ---------------
441 #------- One dimension ---------------
442 if nDims == 0:
442 if nDims == 0:
443 arrayData = dataset.value.astype(numpy.float)[0][blockList]
443 arrayData = dataset.value.astype(numpy.float)[0][blockList]
444
444
445 #------- Two dimensions -----------
445 #------- Two dimensions -----------
446 elif nDims == 2:
446 elif nDims == 2:
447 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
447 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448 newShapes = (blocksPerFile,nDim0)
448 newShapes = (blocksPerFile,nDim0)
449 nDatas = nDim1
449 nDatas = nDim1
450
450
451 for i in range(nDatas):
451 for i in range(nDatas):
452 data = dataset[strds + str(i)].value
452 data = dataset[strds + str(i)].value
453 arrayData[:,i,:] = data[blockList,:]
453 arrayData[:,i,:] = data[blockList,:]
454
454
455 #------- Three dimensions ---------
455 #------- Three dimensions ---------
456 else:
456 else:
457 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
457 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458 for i in range(nDatas):
458 for i in range(nDatas):
459
459
460 data = dataset[strds + str(i)].value
460 data = dataset[strds + str(i)].value
461
461
462 for b in range(blockList.size):
462 for b in range(blockList.size):
463 arrayData[b,:,i,:] = data[:,:,blockList[b]]
463 arrayData[b,:,i,:] = data[:,:,blockList[b]]
464
464
465 return arrayData
465 return arrayData
466
466
467 def __setDataOut(self):
467 def __setDataOut(self):
468 listMeta = self.listMeta
468 listMeta = self.listMeta
469 listMetaname = self.listMetaname
469 listMetaname = self.listMetaname
470 listDataname = self.listDataname
470 listDataname = self.listDataname
471 listData = self.listData
471 listData = self.listData
472 listShapes = self.listShapes
472 listShapes = self.listShapes
473
473
474 blockIndex = self.blockIndex
474 blockIndex = self.blockIndex
475 # blockList = self.blockList
475 # blockList = self.blockList
476
476
477 for i in range(len(listMeta)):
477 for i in range(len(listMeta)):
478 setattr(self.dataOut,listMetaname[i],listMeta[i])
478 setattr(self.dataOut,listMetaname[i],listMeta[i])
479
479
480 for j in range(len(listData)):
480 for j in range(len(listData)):
481 nShapes = listShapes[listDataname[j]][0]
481 nShapes = listShapes[listDataname[j]][0]
482 mode = listShapes[listDataname[j]][4]
482 mode = listShapes[listDataname[j]][4]
483 if nShapes == 1:
483 if nShapes == 1:
484 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
484 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
485 elif nShapes > 1:
485 elif nShapes > 1:
486 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
486 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
487 elif mode==0:
487 elif mode==0:
488 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
488 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
489 #Mode Meteors
489 #Mode Meteors
490 elif mode ==2:
490 elif mode ==2:
491 selectedData = self.__selectDataMode2(listData[j], blockIndex)
491 selectedData = self.__selectDataMode2(listData[j], blockIndex)
492 setattr(self.dataOut, listDataname[j], selectedData)
492 setattr(self.dataOut, listDataname[j], selectedData)
493 return
493 return
494
494
495 def __selectDataMode2(self, data, blockIndex):
495 def __selectDataMode2(self, data, blockIndex):
496 utctime = data[:,0]
496 utctime = data[:,0]
497 aux, indices = numpy.unique(utctime, return_inverse=True)
497 aux, indices = numpy.unique(utctime, return_inverse=True)
498 selInd = numpy.where(indices == blockIndex)[0]
498 selInd = numpy.where(indices == blockIndex)[0]
499 selData = data[selInd,:]
499 selData = data[selInd,:]
500
500
501 return selData
501 return selData
502
502
503 def getData(self):
503 def getData(self):
504
504
505 # if self.flagNoMoreFiles:
505 # if self.flagNoMoreFiles:
506 # self.dataOut.flagNoData = True
506 # self.dataOut.flagNoData = True
507 # print 'Process finished'
507 # print 'Process finished'
508 # return 0
508 # return 0
509 #
509 #
510 if self.blockIndex==self.blocksPerFile:
510 if self.blockIndex==self.blocksPerFile:
511 if not( self.__setNextFileOffline() ):
511 if not( self.__setNextFileOffline() ):
512 self.dataOut.flagNoData = True
512 self.dataOut.flagNoData = True
513 return 0
513 return 0
514
514
515 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
515 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
516 # self.dataOut.flagNoData = True
516 # self.dataOut.flagNoData = True
517 # return 0
517 # return 0
518 # self.__readData()
518 # self.__readData()
519 self.__setDataOut()
519 self.__setDataOut()
520 self.dataOut.flagNoData = False
520 self.dataOut.flagNoData = False
521
521
522 self.blockIndex += 1
522 self.blockIndex += 1
523
523
524 return
524 return
525
525
526 def run(self, **kwargs):
526 def run(self, **kwargs):
527
527
528 if not(self.isConfig):
528 if not(self.isConfig):
529 self.setup(**kwargs)
529 self.setup(**kwargs)
530 # self.setObjProperties()
530 # self.setObjProperties()
531 self.isConfig = True
531 self.isConfig = True
532
532
533 self.getData()
533 self.getData()
534
534
535 return
535 return
536
536
537 class ParamWriter(Operation):
537 class ParamWriter(Operation):
538 '''
538 '''
539 HDF5 Writer, stores parameters data in HDF5 format files
539 HDF5 Writer, stores parameters data in HDF5 format files
540
540
541 path: path where the files will be stored
541 path: path where the files will be stored
542
542
543 blocksPerFile: number of blocks that will be saved in per HDF5 format file
543 blocksPerFile: number of blocks that will be saved in per HDF5 format file
544
544
545 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
545 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
546
546
547 metadataList: list of attributes that will be stored as metadata
547 metadataList: list of attributes that will be stored as metadata
548
548
549 dataList: list of attributes that will be stores as data
549 dataList: list of attributes that will be stores as data
550
550
551 '''
551 '''
552
552
553
553
554 ext = ".hdf5"
554 ext = ".hdf5"
555
555
556 optchar = "D"
556 optchar = "D"
557
557
558 metaoptchar = "M"
558 metaoptchar = "M"
559
559
560 metaFile = None
560 metaFile = None
561
561
562 filename = None
562 filename = None
563
563
564 path = None
564 path = None
565
565
566 setFile = None
566 setFile = None
567
567
568 fp = None
568 fp = None
569
569
570 grp = None
570 grp = None
571
571
572 ds = None
572 ds = None
573
573
574 firsttime = True
574 firsttime = True
575
575
576 #Configurations
576 #Configurations
577
577
578 blocksPerFile = None
578 blocksPerFile = None
579
579
580 blockIndex = None
580 blockIndex = None
581
581
582 dataOut = None
582 dataOut = None
583
583
584 #Data Arrays
584 #Data Arrays
585
585
586 dataList = None
586 dataList = None
587
587
588 metadataList = None
588 metadataList = None
589
589
590 # arrayDim = None
590 # arrayDim = None
591
591
592 dsList = None #List of dictionaries with dataset properties
592 dsList = None #List of dictionaries with dataset properties
593
593
594 tableDim = None
594 tableDim = None
595
595
596 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
596 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
597
597
598 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
598 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
599
599
600 currentDay = None
600 currentDay = None
601
601
602 lastTime = None
603
602 def __init__(self):
604 def __init__(self):
603
605
604 Operation.__init__(self)
606 Operation.__init__(self)
605 self.isConfig = False
607 self.isConfig = False
606 return
608 return
607
609
608 def setup(self, dataOut, **kwargs):
610 def setup(self, dataOut, **kwargs):
609
611
610 self.path = kwargs['path']
612 self.path = kwargs['path']
611
613
612 if kwargs.has_key('blocksPerFile'):
614 if kwargs.has_key('blocksPerFile'):
613 self.blocksPerFile = kwargs['blocksPerFile']
615 self.blocksPerFile = kwargs['blocksPerFile']
614 else:
616 else:
615 self.blocksPerFile = 10
617 self.blocksPerFile = 10
616
618
617 self.metadataList = kwargs['metadataList']
619 self.metadataList = kwargs['metadataList']
618 self.dataList = kwargs['dataList']
620 self.dataList = kwargs['dataList']
619 self.dataOut = dataOut
621 self.dataOut = dataOut
620
622
621 if kwargs.has_key('mode'):
623 if kwargs.has_key('mode'):
622 mode = kwargs['mode']
624 mode = kwargs['mode']
623
625
624 if type(mode) == int:
626 if type(mode) == int:
625 mode = numpy.zeros(len(self.dataList)) + mode
627 mode = numpy.zeros(len(self.dataList)) + mode
626 else:
628 else:
627 mode = numpy.ones(len(self.dataList))
629 mode = numpy.ones(len(self.dataList))
628
630
629 self.mode = mode
631 self.mode = mode
630
632
631 arrayDim = numpy.zeros((len(self.dataList),5))
633 arrayDim = numpy.zeros((len(self.dataList),5))
632
634
633 #Table dimensions
635 #Table dimensions
634 dtype0 = self.dtype
636 dtype0 = self.dtype
635 tableList = []
637 tableList = []
636
638
637 #Dictionary and list of tables
639 #Dictionary and list of tables
638 dsList = []
640 dsList = []
639
641
640 for i in range(len(self.dataList)):
642 for i in range(len(self.dataList)):
641 dsDict = {}
643 dsDict = {}
642 dataAux = getattr(self.dataOut, self.dataList[i])
644 dataAux = getattr(self.dataOut, self.dataList[i])
643 dsDict['variable'] = self.dataList[i]
645 dsDict['variable'] = self.dataList[i]
644 #--------------------- Conditionals ------------------------
646 #--------------------- Conditionals ------------------------
645 #There is no data
647 #There is no data
646 if dataAux == None:
648 if dataAux == None:
647 return 0
649 return 0
648
650
649 #Not array, just a number
651 #Not array, just a number
650 #Mode 0
652 #Mode 0
651 if type(dataAux)==float or type(dataAux)==int:
653 if type(dataAux)==float or type(dataAux)==int:
652 dsDict['mode'] = 0
654 dsDict['mode'] = 0
653 dsDict['nDim'] = 0
655 dsDict['nDim'] = 0
654 arrayDim[i,0] = 0
656 arrayDim[i,0] = 0
655 dsList.append(dsDict)
657 dsList.append(dsDict)
656
658
657 #Mode 2: meteors
659 #Mode 2: meteors
658 elif mode[i] == 2:
660 elif mode[i] == 2:
659 # dsDict['nDim'] = 0
661 # dsDict['nDim'] = 0
660 dsDict['dsName'] = 'table0'
662 dsDict['dsName'] = 'table0'
661 dsDict['mode'] = 2 # Mode meteors
663 dsDict['mode'] = 2 # Mode meteors
662 dsDict['shape'] = dataAux.shape[-1]
664 dsDict['shape'] = dataAux.shape[-1]
663 dsDict['nDim'] = 0
665 dsDict['nDim'] = 0
664 dsDict['dsNumber'] = 1
666 dsDict['dsNumber'] = 1
665
667
666 arrayDim[i,3] = dataAux.shape[-1]
668 arrayDim[i,3] = dataAux.shape[-1]
667 arrayDim[i,4] = mode[i] #Mode the data was stored
669 arrayDim[i,4] = mode[i] #Mode the data was stored
668
670
669 dsList.append(dsDict)
671 dsList.append(dsDict)
670
672
671 #Mode 1
673 #Mode 1
672 else:
674 else:
673 arrayDim0 = dataAux.shape #Data dimensions
675 arrayDim0 = dataAux.shape #Data dimensions
674 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
676 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
675 arrayDim[i,4] = mode[i] #Mode the data was stored
677 arrayDim[i,4] = mode[i] #Mode the data was stored
676
678
677 strtable = 'table'
679 strtable = 'table'
678 dsDict['mode'] = 1 # Mode parameters
680 dsDict['mode'] = 1 # Mode parameters
679
681
680 # Three-dimension arrays
682 # Three-dimension arrays
681 if len(arrayDim0) == 3:
683 if len(arrayDim0) == 3:
682 arrayDim[i,1:-1] = numpy.array(arrayDim0)
684 arrayDim[i,1:-1] = numpy.array(arrayDim0)
683 nTables = int(arrayDim[i,2])
685 nTables = int(arrayDim[i,2])
684 dsDict['dsNumber'] = nTables
686 dsDict['dsNumber'] = nTables
685 dsDict['shape'] = arrayDim[i,2:4]
687 dsDict['shape'] = arrayDim[i,2:4]
686 dsDict['nDim'] = 3
688 dsDict['nDim'] = 3
687
689
688 for j in range(nTables):
690 for j in range(nTables):
689 dsDict = dsDict.copy()
691 dsDict = dsDict.copy()
690 dsDict['dsName'] = strtable + str(j)
692 dsDict['dsName'] = strtable + str(j)
691 dsList.append(dsDict)
693 dsList.append(dsDict)
692
694
693 # Two-dimension arrays
695 # Two-dimension arrays
694 elif len(arrayDim0) == 2:
696 elif len(arrayDim0) == 2:
695 arrayDim[i,2:-1] = numpy.array(arrayDim0)
697 arrayDim[i,2:-1] = numpy.array(arrayDim0)
696 nTables = int(arrayDim[i,2])
698 nTables = int(arrayDim[i,2])
697 dsDict['dsNumber'] = nTables
699 dsDict['dsNumber'] = nTables
698 dsDict['shape'] = arrayDim[i,3]
700 dsDict['shape'] = arrayDim[i,3]
699 dsDict['nDim'] = 2
701 dsDict['nDim'] = 2
700
702
701 for j in range(nTables):
703 for j in range(nTables):
702 dsDict = dsDict.copy()
704 dsDict = dsDict.copy()
703 dsDict['dsName'] = strtable + str(j)
705 dsDict['dsName'] = strtable + str(j)
704 dsList.append(dsDict)
706 dsList.append(dsDict)
705
707
706 # One-dimension arrays
708 # One-dimension arrays
707 elif len(arrayDim0) == 1:
709 elif len(arrayDim0) == 1:
708 arrayDim[i,3] = arrayDim0[0]
710 arrayDim[i,3] = arrayDim0[0]
709 dsDict['shape'] = arrayDim0[0]
711 dsDict['shape'] = arrayDim0[0]
710 dsDict['dsNumber'] = 1
712 dsDict['dsNumber'] = 1
711 dsDict['dsName'] = strtable + str(0)
713 dsDict['dsName'] = strtable + str(0)
712 dsDict['nDim'] = 1
714 dsDict['nDim'] = 1
713 dsList.append(dsDict)
715 dsList.append(dsDict)
714
716
715 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
717 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
716 tableList.append(table)
718 tableList.append(table)
717
719
718 # self.arrayDim = arrayDim
720 # self.arrayDim = arrayDim
719 self.dsList = dsList
721 self.dsList = dsList
720 self.tableDim = numpy.array(tableList, dtype = dtype0)
722 self.tableDim = numpy.array(tableList, dtype = dtype0)
721 self.blockIndex = 0
723 self.blockIndex = 0
722
724
723 timeTuple = time.localtime(dataOut.utctime)
725 timeTuple = time.localtime(dataOut.utctime)
724 self.currentDay = timeTuple.tm_yday
726 self.currentDay = timeTuple.tm_yday
725 return 1
727 return 1
726
728
727 def putMetadata(self):
729 def putMetadata(self):
728
730
729 fp = self.createMetadataFile()
731 fp = self.createMetadataFile()
730 self.writeMetadata(fp)
732 self.writeMetadata(fp)
731 fp.close()
733 fp.close()
732 return
734 return
733
735
734 def createMetadataFile(self):
736 def createMetadataFile(self):
735 ext = self.ext
737 ext = self.ext
736 path = self.path
738 path = self.path
737 setFile = self.setFile
739 setFile = self.setFile
738
740
739 timeTuple = time.localtime(self.dataOut.utctime)
741 timeTuple = time.localtime(self.dataOut.utctime)
740
742
741 subfolder = ''
743 subfolder = ''
742 fullpath = os.path.join( path, subfolder )
744 fullpath = os.path.join( path, subfolder )
743
745
744 if not( os.path.exists(fullpath) ):
746 if not( os.path.exists(fullpath) ):
745 os.mkdir(fullpath)
747 os.mkdir(fullpath)
746 setFile = -1 #inicializo mi contador de seteo
748 setFile = -1 #inicializo mi contador de seteo
747
749
748 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
750 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
749 fullpath = os.path.join( path, subfolder )
751 fullpath = os.path.join( path, subfolder )
750
752
751 if not( os.path.exists(fullpath) ):
753 if not( os.path.exists(fullpath) ):
752 os.mkdir(fullpath)
754 os.mkdir(fullpath)
753 setFile = -1 #inicializo mi contador de seteo
755 setFile = -1 #inicializo mi contador de seteo
754
756
755 else:
757 else:
756 filesList = os.listdir( fullpath )
758 filesList = os.listdir( fullpath )
757 filesList = sorted( filesList, key=str.lower )
759 filesList = sorted( filesList, key=str.lower )
758 if len( filesList ) > 0:
760 if len( filesList ) > 0:
759 filesList = [k for k in filesList if 'M' in k]
761 filesList = [k for k in filesList if 'M' in k]
760 filen = filesList[-1]
762 filen = filesList[-1]
761 # el filename debera tener el siguiente formato
763 # el filename debera tener el siguiente formato
762 # 0 1234 567 89A BCDE (hex)
764 # 0 1234 567 89A BCDE (hex)
763 # x YYYY DDD SSS .ext
765 # x YYYY DDD SSS .ext
764 if isNumber( filen[8:11] ):
766 if isNumber( filen[8:11] ):
765 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
767 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
766 else:
768 else:
767 setFile = -1
769 setFile = -1
768 else:
770 else:
769 setFile = -1 #inicializo mi contador de seteo
771 setFile = -1 #inicializo mi contador de seteo
770
772
771 setFile += 1
773 setFile += 1
772
774
773 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
775 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
774 timeTuple.tm_year,
776 timeTuple.tm_year,
775 timeTuple.tm_yday,
777 timeTuple.tm_yday,
776 setFile,
778 setFile,
777 ext )
779 ext )
778
780
779 filename = os.path.join( path, subfolder, file )
781 filename = os.path.join( path, subfolder, file )
780 self.metaFile = file
782 self.metaFile = file
781 #Setting HDF5 File
783 #Setting HDF5 File
782 fp = h5py.File(filename,'w')
784 fp = h5py.File(filename,'w')
783
785
784 return fp
786 return fp
785
787
786 def writeMetadata(self, fp):
788 def writeMetadata(self, fp):
787
789
788 grp = fp.create_group("Metadata")
790 grp = fp.create_group("Metadata")
789 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
791 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
790
792
791 for i in range(len(self.metadataList)):
793 for i in range(len(self.metadataList)):
792 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
794 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
793 return
795 return
794
796
795 def dateFlag(self):
797 def timeFlag(self):
798 currentTime = self.dataOut.utctime
796
799
797 timeTuple = time.localtime(self.dataOut.utctime)
800 if self.lastTime is None:
801 self.lastTime = currentTime
802
803 #Day
804 timeTuple = time.localtime(currentTime)
798 dataDay = timeTuple.tm_yday
805 dataDay = timeTuple.tm_yday
799
806
800 if dataDay == self.currentDay:
807 #Time
801 return False
808 timeDiff = currentTime - self.lastTime
802
809
803 self.currentDay = dataDay
810 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
804 return True
811 if dataDay != self.currentDay:
812 self.currentDay = dataDay
813 return True
814 elif timeDiff > 3*60*60:
815 self.lastTime = currentTime
816 return True
817 else:
818 self.lastTime = currentTime
819 return False
805
820
806 def setNextFile(self):
821 def setNextFile(self):
807
822
808 ext = self.ext
823 ext = self.ext
809 path = self.path
824 path = self.path
810 setFile = self.setFile
825 setFile = self.setFile
811 mode = self.mode
826 mode = self.mode
812
827
813 timeTuple = time.localtime(self.dataOut.utctime)
828 timeTuple = time.localtime(self.dataOut.utctime)
814 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
829 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
815
830
816 fullpath = os.path.join( path, subfolder )
831 fullpath = os.path.join( path, subfolder )
817
832
818 if os.path.exists(fullpath):
833 if os.path.exists(fullpath):
819 filesList = os.listdir( fullpath )
834 filesList = os.listdir( fullpath )
820 filesList = [k for k in filesList if 'D' in k]
835 filesList = [k for k in filesList if 'D' in k]
821 if len( filesList ) > 0:
836 if len( filesList ) > 0:
822 filesList = sorted( filesList, key=str.lower )
837 filesList = sorted( filesList, key=str.lower )
823 filen = filesList[-1]
838 filen = filesList[-1]
824 # el filename debera tener el siguiente formato
839 # el filename debera tener el siguiente formato
825 # 0 1234 567 89A BCDE (hex)
840 # 0 1234 567 89A BCDE (hex)
826 # x YYYY DDD SSS .ext
841 # x YYYY DDD SSS .ext
827 if isNumber( filen[8:11] ):
842 if isNumber( filen[8:11] ):
828 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
843 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
829 else:
844 else:
830 setFile = -1
845 setFile = -1
831 else:
846 else:
832 setFile = -1 #inicializo mi contador de seteo
847 setFile = -1 #inicializo mi contador de seteo
833 else:
848 else:
834 os.mkdir(fullpath)
849 os.mkdir(fullpath)
835 setFile = -1 #inicializo mi contador de seteo
850 setFile = -1 #inicializo mi contador de seteo
836
851
837 setFile += 1
852 setFile += 1
838
853
839 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
854 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
840 timeTuple.tm_year,
855 timeTuple.tm_year,
841 timeTuple.tm_yday,
856 timeTuple.tm_yday,
842 setFile,
857 setFile,
843 ext )
858 ext )
844
859
845 filename = os.path.join( path, subfolder, file )
860 filename = os.path.join( path, subfolder, file )
846
861
847 #Setting HDF5 File
862 #Setting HDF5 File
848 fp = h5py.File(filename,'w')
863 fp = h5py.File(filename,'w')
849 #write metadata
864 #write metadata
850 self.writeMetadata(fp)
865 self.writeMetadata(fp)
851 #Write data
866 #Write data
852 grp = fp.create_group("Data")
867 grp = fp.create_group("Data")
853 # grp.attrs['metadata'] = self.metaFile
868 # grp.attrs['metadata'] = self.metaFile
854
869
855 # grp.attrs['blocksPerFile'] = 0
870 # grp.attrs['blocksPerFile'] = 0
856 ds = []
871 ds = []
857 data = []
872 data = []
858 dsList = self.dsList
873 dsList = self.dsList
859 i = 0
874 i = 0
860 while i < len(dsList):
875 while i < len(dsList):
861 dsInfo = dsList[i]
876 dsInfo = dsList[i]
862 #One-dimension data
877 #One-dimension data
863 if dsInfo['mode'] == 0:
878 if dsInfo['mode'] == 0:
864 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
879 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
865 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
880 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
866 ds.append(ds0)
881 ds.append(ds0)
867 data.append([])
882 data.append([])
868 i += 1
883 i += 1
869 continue
884 continue
870 # nDimsForDs.append(nDims[i])
885 # nDimsForDs.append(nDims[i])
871
886
872 elif dsInfo['mode'] == 2:
887 elif dsInfo['mode'] == 2:
873 grp0 = grp.create_group(dsInfo['variable'])
888 grp0 = grp.create_group(dsInfo['variable'])
874 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
889 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
875 ds.append(ds0)
890 ds.append(ds0)
876 data.append([])
891 data.append([])
877 i += 1
892 i += 1
878 continue
893 continue
879
894
880 elif dsInfo['mode'] == 1:
895 elif dsInfo['mode'] == 1:
881 grp0 = grp.create_group(dsInfo['variable'])
896 grp0 = grp.create_group(dsInfo['variable'])
882
897
883 for j in range(dsInfo['dsNumber']):
898 for j in range(dsInfo['dsNumber']):
884 dsInfo = dsList[i]
899 dsInfo = dsList[i]
885 tableName = dsInfo['dsName']
900 tableName = dsInfo['dsName']
886 shape = dsInfo['shape']
901 shape = dsInfo['shape']
887
902
888 if dsInfo['nDim'] == 3:
903 if dsInfo['nDim'] == 3:
889 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
904 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
890 else:
905 else:
891 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
906 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
892
907
893 ds.append(ds0)
908 ds.append(ds0)
894 data.append([])
909 data.append([])
895 i += 1
910 i += 1
896 # nDimsForDs.append(nDims[i])
911 # nDimsForDs.append(nDims[i])
897
912
898 fp.flush()
913 fp.flush()
899 fp.close()
914 fp.close()
900
915
901 # self.nDatas = nDatas
916 # self.nDatas = nDatas
902 # self.nDims = nDims
917 # self.nDims = nDims
903 # self.nDimsForDs = nDimsForDs
918 # self.nDimsForDs = nDimsForDs
904 #Saving variables
919 #Saving variables
905 print 'Writing the file: %s'%filename
920 print 'Writing the file: %s'%filename
906 self.filename = filename
921 self.filename = filename
907 # self.fp = fp
922 # self.fp = fp
908 # self.grp = grp
923 # self.grp = grp
909 # self.grp.attrs.modify('nRecords', 1)
924 # self.grp.attrs.modify('nRecords', 1)
910 self.ds = ds
925 self.ds = ds
911 self.data = data
926 self.data = data
912 # self.setFile = setFile
927 # self.setFile = setFile
913 self.firsttime = True
928 self.firsttime = True
914 self.blockIndex = 0
929 self.blockIndex = 0
915 return
930 return
916
931
917 def putData(self):
932 def putData(self):
918
933
919 if self.blockIndex == self.blocksPerFile or self.dateFlag():
934 if self.blockIndex == self.blocksPerFile or self.timeFlag():
920 self.setNextFile()
935 self.setNextFile()
921
936
922 # if not self.firsttime:
937 # if not self.firsttime:
923 self.readBlock()
938 self.readBlock()
924 self.setBlock() #Prepare data to be written
939 self.setBlock() #Prepare data to be written
925 self.writeBlock() #Write data
940 self.writeBlock() #Write data
926
941
927 return
942 return
928
943
929 def readBlock(self):
944 def readBlock(self):
930
945
931 '''
946 '''
932 data Array configured
947 data Array configured
933
948
934
949
935 self.data
950 self.data
936 '''
951 '''
937 dsList = self.dsList
952 dsList = self.dsList
938 ds = self.ds
953 ds = self.ds
939 #Setting HDF5 File
954 #Setting HDF5 File
940 fp = h5py.File(self.filename,'r+')
955 fp = h5py.File(self.filename,'r+')
941 grp = fp["Data"]
956 grp = fp["Data"]
942 ind = 0
957 ind = 0
943
958
944 # grp.attrs['blocksPerFile'] = 0
959 # grp.attrs['blocksPerFile'] = 0
945 while ind < len(dsList):
960 while ind < len(dsList):
946 dsInfo = dsList[ind]
961 dsInfo = dsList[ind]
947
962
948 if dsInfo['mode'] == 0:
963 if dsInfo['mode'] == 0:
949 ds0 = grp[dsInfo['variable']]
964 ds0 = grp[dsInfo['variable']]
950 ds[ind] = ds0
965 ds[ind] = ds0
951 ind += 1
966 ind += 1
952 else:
967 else:
953
968
954 grp0 = grp[dsInfo['variable']]
969 grp0 = grp[dsInfo['variable']]
955
970
956 for j in range(dsInfo['dsNumber']):
971 for j in range(dsInfo['dsNumber']):
957 dsInfo = dsList[ind]
972 dsInfo = dsList[ind]
958 ds0 = grp0[dsInfo['dsName']]
973 ds0 = grp0[dsInfo['dsName']]
959 ds[ind] = ds0
974 ds[ind] = ds0
960 ind += 1
975 ind += 1
961
976
962 self.fp = fp
977 self.fp = fp
963 self.grp = grp
978 self.grp = grp
964 self.ds = ds
979 self.ds = ds
965
980
966 return
981 return
967
982
968 def setBlock(self):
983 def setBlock(self):
969 '''
984 '''
970 data Array configured
985 data Array configured
971
986
972
987
973 self.data
988 self.data
974 '''
989 '''
975 #Creating Arrays
990 #Creating Arrays
976 dsList = self.dsList
991 dsList = self.dsList
977 data = self.data
992 data = self.data
978 ind = 0
993 ind = 0
979
994
980 while ind < len(dsList):
995 while ind < len(dsList):
981 dsInfo = dsList[ind]
996 dsInfo = dsList[ind]
982 dataAux = getattr(self.dataOut, dsInfo['variable'])
997 dataAux = getattr(self.dataOut, dsInfo['variable'])
983
998
984 mode = dsInfo['mode']
999 mode = dsInfo['mode']
985 nDim = dsInfo['nDim']
1000 nDim = dsInfo['nDim']
986
1001
987 if mode == 0 or mode == 2 or nDim == 1:
1002 if mode == 0 or mode == 2 or nDim == 1:
988 data[ind] = dataAux
1003 data[ind] = dataAux
989 ind += 1
1004 ind += 1
990 # elif nDim == 1:
1005 # elif nDim == 1:
991 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1006 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
992 # ind += 1
1007 # ind += 1
993 elif nDim == 2:
1008 elif nDim == 2:
994 for j in range(dsInfo['dsNumber']):
1009 for j in range(dsInfo['dsNumber']):
995 data[ind] = dataAux[j,:]
1010 data[ind] = dataAux[j,:]
996 ind += 1
1011 ind += 1
997 elif nDim == 3:
1012 elif nDim == 3:
998 for j in range(dsInfo['dsNumber']):
1013 for j in range(dsInfo['dsNumber']):
999 data[ind] = dataAux[:,j,:]
1014 data[ind] = dataAux[:,j,:]
1000 ind += 1
1015 ind += 1
1001
1016
1002 self.data = data
1017 self.data = data
1003 return
1018 return
1004
1019
1005 def writeBlock(self):
1020 def writeBlock(self):
1006 '''
1021 '''
1007 Saves the block in the HDF5 file
1022 Saves the block in the HDF5 file
1008 '''
1023 '''
1009 dsList = self.dsList
1024 dsList = self.dsList
1010
1025
1011 for i in range(len(self.ds)):
1026 for i in range(len(self.ds)):
1012 dsInfo = dsList[i]
1027 dsInfo = dsList[i]
1013 nDim = dsInfo['nDim']
1028 nDim = dsInfo['nDim']
1014 mode = dsInfo['mode']
1029 mode = dsInfo['mode']
1015
1030
1016 # First time
1031 # First time
1017 if self.firsttime:
1032 if self.firsttime:
1018 # self.ds[i].resize(self.data[i].shape)
1033 # self.ds[i].resize(self.data[i].shape)
1019 # self.ds[i][self.blockIndex,:] = self.data[i]
1034 # self.ds[i][self.blockIndex,:] = self.data[i]
1020 if type(self.data[i]) == numpy.ndarray:
1035 if type(self.data[i]) == numpy.ndarray:
1021
1036
1022 if nDim == 3:
1037 if nDim == 3:
1023 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1038 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1024 self.ds[i].resize(self.data[i].shape)
1039 self.ds[i].resize(self.data[i].shape)
1025 if mode == 2:
1040 if mode == 2:
1026 self.ds[i].resize(self.data[i].shape)
1041 self.ds[i].resize(self.data[i].shape)
1027 self.ds[i][:] = self.data[i]
1042 self.ds[i][:] = self.data[i]
1028 else:
1043 else:
1029
1044
1030 # From second time
1045 # From second time
1031 # Meteors!
1046 # Meteors!
1032 if mode == 2:
1047 if mode == 2:
1033 dataShape = self.data[i].shape
1048 dataShape = self.data[i].shape
1034 dsShape = self.ds[i].shape
1049 dsShape = self.ds[i].shape
1035 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1050 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1036 self.ds[i][dsShape[0]:,:] = self.data[i]
1051 self.ds[i][dsShape[0]:,:] = self.data[i]
1037 # No dimension
1052 # No dimension
1038 elif mode == 0:
1053 elif mode == 0:
1039 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1054 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1040 self.ds[i][0,-1] = self.data[i]
1055 self.ds[i][0,-1] = self.data[i]
1041 # One dimension
1056 # One dimension
1042 elif nDim == 1:
1057 elif nDim == 1:
1043 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1058 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1044 self.ds[i][-1,:] = self.data[i]
1059 self.ds[i][-1,:] = self.data[i]
1045 # Two dimension
1060 # Two dimension
1046 elif nDim == 2:
1061 elif nDim == 2:
1047 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1062 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1048 self.ds[i][self.blockIndex,:] = self.data[i]
1063 self.ds[i][self.blockIndex,:] = self.data[i]
1049 # Three dimensions
1064 # Three dimensions
1050 elif nDim == 3:
1065 elif nDim == 3:
1051 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1066 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1052 self.ds[i][:,:,-1] = self.data[i]
1067 self.ds[i][:,:,-1] = self.data[i]
1053
1068
1054 self.firsttime = False
1069 self.firsttime = False
1055 self.blockIndex += 1
1070 self.blockIndex += 1
1056
1071
1057 #Close to save changes
1072 #Close to save changes
1058 self.fp.flush()
1073 self.fp.flush()
1059 self.fp.close()
1074 self.fp.close()
1060 return
1075 return
1061
1076
1062 def run(self, dataOut, **kwargs):
1077 def run(self, dataOut, **kwargs):
1063
1078
1064 if not(self.isConfig):
1079 if not(self.isConfig):
1065 flagdata = self.setup(dataOut, **kwargs)
1080 flagdata = self.setup(dataOut, **kwargs)
1066
1081
1067 if not(flagdata):
1082 if not(flagdata):
1068 return
1083 return
1069
1084
1070 self.isConfig = True
1085 self.isConfig = True
1071 # self.putMetadata()
1086 # self.putMetadata()
1072 self.setNextFile()
1087 self.setNextFile()
1073
1088
1074 self.putData()
1089 self.putData()
1075 return
1090 return
1076
1091
1077
1092
General Comments 0
You need to be logged in to leave comments. Login now