##// END OF EJS Templates
schain-jc
RSS
Add setType to ParamWriter
Juan C. Espinoza -
r986:0e8ea66dee39
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1090 +1,1105
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class ParamReader(ProcessingUnit):
15 class ParamReader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self, **kwargs):
77 def __init__(self, **kwargs):
78 ProcessingUnit.__init__(self, **kwargs)
78 ProcessingUnit.__init__(self, **kwargs)
79 self.dataOut = Parameters()
79 self.dataOut = Parameters()
80 return
80 return
81
81
82 def setup(self, **kwargs):
82 def setup(self, **kwargs):
83
83
84 path = kwargs['path']
84 path = kwargs['path']
85 startDate = kwargs['startDate']
85 startDate = kwargs['startDate']
86 endDate = kwargs['endDate']
86 endDate = kwargs['endDate']
87 startTime = kwargs['startTime']
87 startTime = kwargs['startTime']
88 endTime = kwargs['endTime']
88 endTime = kwargs['endTime']
89 walk = kwargs['walk']
89 walk = kwargs['walk']
90 if kwargs.has_key('ext'):
90 if kwargs.has_key('ext'):
91 ext = kwargs['ext']
91 ext = kwargs['ext']
92 else:
92 else:
93 ext = '.hdf5'
93 ext = '.hdf5'
94 if kwargs.has_key('timezone'):
94 if kwargs.has_key('timezone'):
95 self.timezone = kwargs['timezone']
95 self.timezone = kwargs['timezone']
96 else:
96 else:
97 self.timezone = 'lt'
97 self.timezone = 'lt'
98
98
99 print "[Reading] Searching files in offline mode ..."
99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 startTime=startTime, endTime=endTime,
101 startTime=startTime, endTime=endTime,
102 ext=ext, walk=walk)
102 ext=ext, walk=walk)
103
103
104 if not(filenameList):
104 if not(filenameList):
105 print "There is no files into the folder: %s"%(path)
105 print "There is no files into the folder: %s"%(path)
106 sys.exit(-1)
106 sys.exit(-1)
107
107
108 self.fileIndex = -1
108 self.fileIndex = -1
109 self.startTime = startTime
109 self.startTime = startTime
110 self.endTime = endTime
110 self.endTime = endTime
111
111
112 self.__readMetadata()
112 self.__readMetadata()
113
113
114 self.__setNextFileOffline()
114 self.__setNextFileOffline()
115
115
116 return
116 return
117
117
118 def __searchFilesOffLine(self,
118 def __searchFilesOffLine(self,
119 path,
119 path,
120 startDate=None,
120 startDate=None,
121 endDate=None,
121 endDate=None,
122 startTime=datetime.time(0,0,0),
122 startTime=datetime.time(0,0,0),
123 endTime=datetime.time(23,59,59),
123 endTime=datetime.time(23,59,59),
124 ext='.hdf5',
124 ext='.hdf5',
125 walk=True):
125 walk=True):
126
126
127 expLabel = ''
127 expLabel = ''
128 self.filenameList = []
128 self.filenameList = []
129 self.datetimeList = []
129 self.datetimeList = []
130
130
131 pathList = []
131 pathList = []
132
132
133 JRODataObj = JRODataReader()
133 JRODataObj = JRODataReader()
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135
135
136 if dateList == []:
136 if dateList == []:
137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 datetime.datetime.combine(startDate,startTime).ctime(),
138 datetime.datetime.combine(startDate,startTime).ctime(),
139 datetime.datetime.combine(endDate,endTime).ctime())
139 datetime.datetime.combine(endDate,endTime).ctime())
140
140
141 return None, None
141 return None, None
142
142
143 if len(dateList) > 1:
143 if len(dateList) > 1:
144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
145 else:
145 else:
146 print "[Reading] data was found for the date %s" %(dateList[0])
146 print "[Reading] data was found for the date %s" %(dateList[0])
147
147
148 filenameList = []
148 filenameList = []
149 datetimeList = []
149 datetimeList = []
150
150
151 #----------------------------------------------------------------------------------
151 #----------------------------------------------------------------------------------
152
152
153 for thisPath in pathList:
153 for thisPath in pathList:
154 # thisPath = pathList[pathDict[file]]
154 # thisPath = pathList[pathDict[file]]
155
155
156 fileList = glob.glob1(thisPath, "*%s" %ext)
156 fileList = glob.glob1(thisPath, "*%s" %ext)
157 fileList.sort()
157 fileList.sort()
158
158
159 for file in fileList:
159 for file in fileList:
160
160
161 filename = os.path.join(thisPath,file)
161 filename = os.path.join(thisPath,file)
162
162
163 if not isFileInDateRange(filename, startDate, endDate):
163 if not isFileInDateRange(filename, startDate, endDate):
164 continue
164 continue
165
165
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167
167
168 if not(thisDatetime):
168 if not(thisDatetime):
169 continue
169 continue
170
170
171 filenameList.append(filename)
171 filenameList.append(filename)
172 datetimeList.append(thisDatetime)
172 datetimeList.append(thisDatetime)
173
173
174 if not(filenameList):
174 if not(filenameList):
175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
176 return None, None
176 return None, None
177
177
178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
179 print
179 print
180
180
181 for i in range(len(filenameList)):
181 for i in range(len(filenameList)):
182 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
182 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183
183
184 self.filenameList = filenameList
184 self.filenameList = filenameList
185 self.datetimeList = datetimeList
185 self.datetimeList = datetimeList
186
186
187 return pathList, filenameList
187 return pathList, filenameList
188
188
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190
190
191 """
191 """
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193
193
194 Inputs:
194 Inputs:
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196
196
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
198
198
199 endDate : fecha final del rango seleccionado en formato datetime.date
199 endDate : fecha final del rango seleccionado en formato datetime.date
200
200
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202
202
203 endTime : tiempo final del rango seleccionado en formato datetime.time
203 endTime : tiempo final del rango seleccionado en formato datetime.time
204
204
205 Return:
205 Return:
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 fecha especificado, de lo contrario retorna False.
207 fecha especificado, de lo contrario retorna False.
208
208
209 Excepciones:
209 Excepciones:
210 Si el archivo no existe o no puede ser abierto
210 Si el archivo no existe o no puede ser abierto
211 Si la cabecera no puede ser leida.
211 Si la cabecera no puede ser leida.
212
212
213 """
213 """
214
214
215 try:
215 try:
216 fp = h5py.File(filename,'r')
216 fp = h5py.File(filename,'r')
217 grp1 = fp['Data']
217 grp1 = fp['Data']
218
218
219 except IOError:
219 except IOError:
220 traceback.print_exc()
220 traceback.print_exc()
221 raise IOError, "The file %s can't be opened" %(filename)
221 raise IOError, "The file %s can't be opened" %(filename)
222 #chino rata
222 #chino rata
223 #In case has utctime attribute
223 #In case has utctime attribute
224 grp2 = grp1['utctime']
224 grp2 = grp1['utctime']
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 thisUtcTime = grp2.value[0]
226 thisUtcTime = grp2.value[0]
227
227
228 fp.close()
228 fp.close()
229
229
230 if self.timezone == 'lt':
230 if self.timezone == 'lt':
231 thisUtcTime -= 5*3600
231 thisUtcTime -= 5*3600
232
232
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 thisDate = thisDatetime.date()
235 thisDate = thisDatetime.date()
236 thisTime = thisDatetime.time()
236 thisTime = thisDatetime.time()
237
237
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240
240
241 #General case
241 #General case
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 #-----------o----------------------------o-----------
243 #-----------o----------------------------o-----------
244 # startTime endTime
244 # startTime endTime
245
245
246 if endTime >= startTime:
246 if endTime >= startTime:
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 return thisDatetime
249 return thisDatetime
250 return None
250 return None
251
251
252 #If endTime < startTime then endTime belongs to the next day
252 #If endTime < startTime then endTime belongs to the next day
253 #<<<<<<<<<<<o o>>>>>>>>>>>
253 #<<<<<<<<<<<o o>>>>>>>>>>>
254 #-----------o----------------------------o-----------
254 #-----------o----------------------------o-----------
255 # endTime startTime
255 # endTime startTime
256
256
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 return None
258 return None
259
259
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 return None
261 return None
262
262
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 return None
264 return None
265
265
266 return thisDatetime
266 return thisDatetime
267
267
268 def __setNextFileOffline(self):
268 def __setNextFileOffline(self):
269
269
270 self.fileIndex += 1
270 self.fileIndex += 1
271 idFile = self.fileIndex
271 idFile = self.fileIndex
272
272
273 if not(idFile < len(self.filenameList)):
273 if not(idFile < len(self.filenameList)):
274 print "No more Files"
274 print "No more Files"
275 return 0
275 return 0
276
276
277 filename = self.filenameList[idFile]
277 filename = self.filenameList[idFile]
278
278
279 filePointer = h5py.File(filename,'r')
279 filePointer = h5py.File(filename,'r')
280
280
281 self.filename = filename
281 self.filename = filename
282
282
283 self.fp = filePointer
283 self.fp = filePointer
284
284
285 print "Setting the file: %s"%self.filename
285 print "Setting the file: %s"%self.filename
286
286
287 # self.__readMetadata()
287 # self.__readMetadata()
288 self.__setBlockList()
288 self.__setBlockList()
289 self.__readData()
289 self.__readData()
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 self.blockIndex = 0
292 self.blockIndex = 0
293 return 1
293 return 1
294
294
295 def __setBlockList(self):
295 def __setBlockList(self):
296 '''
296 '''
297 Selects the data within the times defined
297 Selects the data within the times defined
298
298
299 self.fp
299 self.fp
300 self.startTime
300 self.startTime
301 self.endTime
301 self.endTime
302
302
303 self.blockList
303 self.blockList
304 self.blocksPerFile
304 self.blocksPerFile
305
305
306 '''
306 '''
307 fp = self.fp
307 fp = self.fp
308 startTime = self.startTime
308 startTime = self.startTime
309 endTime = self.endTime
309 endTime = self.endTime
310
310
311 grp = fp['Data']
311 grp = fp['Data']
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313
313
314 #ERROOOOR
314 #ERROOOOR
315 if self.timezone == 'lt':
315 if self.timezone == 'lt':
316 thisUtcTime -= 5*3600
316 thisUtcTime -= 5*3600
317
317
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319
319
320 thisDate = thisDatetime.date()
320 thisDate = thisDatetime.date()
321 thisTime = thisDatetime.time()
321 thisTime = thisDatetime.time()
322
322
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325
325
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327
327
328 self.blockList = ind
328 self.blockList = ind
329 self.blocksPerFile = len(ind)
329 self.blocksPerFile = len(ind)
330
330
331 return
331 return
332
332
333 def __readMetadata(self):
333 def __readMetadata(self):
334 '''
334 '''
335 Reads Metadata
335 Reads Metadata
336
336
337 self.pathMeta
337 self.pathMeta
338
338
339 self.listShapes
339 self.listShapes
340 self.listMetaname
340 self.listMetaname
341 self.listMeta
341 self.listMeta
342
342
343 '''
343 '''
344
344
345 # grp = self.fp['Data']
345 # grp = self.fp['Data']
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 #
347 #
348 # if pathMeta == self.pathMeta:
348 # if pathMeta == self.pathMeta:
349 # return
349 # return
350 # else:
350 # else:
351 # self.pathMeta = pathMeta
351 # self.pathMeta = pathMeta
352 #
352 #
353 # filePointer = h5py.File(self.pathMeta,'r')
353 # filePointer = h5py.File(self.pathMeta,'r')
354 # groupPointer = filePointer['Metadata']
354 # groupPointer = filePointer['Metadata']
355
355
356 filename = self.filenameList[0]
356 filename = self.filenameList[0]
357
357
358 fp = h5py.File(filename,'r')
358 fp = h5py.File(filename,'r')
359
359
360 gp = fp['Metadata']
360 gp = fp['Metadata']
361
361
362 listMetaname = []
362 listMetaname = []
363 listMetadata = []
363 listMetadata = []
364 for item in gp.items():
364 for item in gp.items():
365 name = item[0]
365 name = item[0]
366
366
367 if name=='array dimensions':
367 if name=='array dimensions':
368 table = gp[name][:]
368 table = gp[name][:]
369 listShapes = {}
369 listShapes = {}
370 for shapes in table:
370 for shapes in table:
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 else:
372 else:
373 data = gp[name].value
373 data = gp[name].value
374 listMetaname.append(name)
374 listMetaname.append(name)
375 listMetadata.append(data)
375 listMetadata.append(data)
376
376
377 # if name=='type':
377 # if name=='type':
378 # self.__initDataOut(data)
378 # self.__initDataOut(data)
379
379
380 self.listShapes = listShapes
380 self.listShapes = listShapes
381 self.listMetaname = listMetaname
381 self.listMetaname = listMetaname
382 self.listMeta = listMetadata
382 self.listMeta = listMetadata
383
383
384 fp.close()
384 fp.close()
385 return
385 return
386
386
387 def __readData(self):
387 def __readData(self):
388 grp = self.fp['Data']
388 grp = self.fp['Data']
389 listdataname = []
389 listdataname = []
390 listdata = []
390 listdata = []
391
391
392 for item in grp.items():
392 for item in grp.items():
393 name = item[0]
393 name = item[0]
394 listdataname.append(name)
394 listdataname.append(name)
395
395
396 array = self.__setDataArray(grp[name],self.listShapes[name])
396 array = self.__setDataArray(grp[name],self.listShapes[name])
397 listdata.append(array)
397 listdata.append(array)
398
398
399 self.listDataname = listdataname
399 self.listDataname = listdataname
400 self.listData = listdata
400 self.listData = listdata
401 return
401 return
402
402
403 def __setDataArray(self, dataset, shapes):
403 def __setDataArray(self, dataset, shapes):
404
404
405 nDims = shapes[0]
405 nDims = shapes[0]
406
406
407 nDim2 = shapes[1] #Dimension 0
407 nDim2 = shapes[1] #Dimension 0
408
408
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410
410
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412
412
413 mode = shapes[4] #Mode of storing
413 mode = shapes[4] #Mode of storing
414
414
415 blockList = self.blockList
415 blockList = self.blockList
416
416
417 blocksPerFile = self.blocksPerFile
417 blocksPerFile = self.blocksPerFile
418
418
419 #Depending on what mode the data was stored
419 #Depending on what mode the data was stored
420 if mode == 0: #Divided in channels
420 if mode == 0: #Divided in channels
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 if mode == 1: #Divided in parameter
422 if mode == 1: #Divided in parameter
423 strds = 'table'
423 strds = 'table'
424 nDatas = nDim1
424 nDatas = nDim1
425 newShapes = (blocksPerFile,nDim2,nDim0)
425 newShapes = (blocksPerFile,nDim2,nDim0)
426 elif mode==2: #Concatenated in a table
426 elif mode==2: #Concatenated in a table
427 strds = 'table0'
427 strds = 'table0'
428 arrayData = dataset[strds].value
428 arrayData = dataset[strds].value
429 #Selecting part of the dataset
429 #Selecting part of the dataset
430 utctime = arrayData[:,0]
430 utctime = arrayData[:,0]
431 u, indices = numpy.unique(utctime, return_index=True)
431 u, indices = numpy.unique(utctime, return_index=True)
432
432
433 if blockList.size != indices.size:
433 if blockList.size != indices.size:
434 indMin = indices[blockList[0]]
434 indMin = indices[blockList[0]]
435 if blockList[1] + 1 >= indices.size:
435 if blockList[1] + 1 >= indices.size:
436 arrayData = arrayData[indMin:,:]
436 arrayData = arrayData[indMin:,:]
437 else:
437 else:
438 indMax = indices[blockList[1] + 1]
438 indMax = indices[blockList[1] + 1]
439 arrayData = arrayData[indMin:indMax,:]
439 arrayData = arrayData[indMin:indMax,:]
440 return arrayData
440 return arrayData
441
441
442 # One dimension
442 # One dimension
443 if nDims == 0:
443 if nDims == 0:
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445
445
446 # Two dimensions
446 # Two dimensions
447 elif nDims == 2:
447 elif nDims == 2:
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 newShapes = (blocksPerFile,nDim0)
449 newShapes = (blocksPerFile,nDim0)
450 nDatas = nDim1
450 nDatas = nDim1
451
451
452 for i in range(nDatas):
452 for i in range(nDatas):
453 data = dataset[strds + str(i)].value
453 data = dataset[strds + str(i)].value
454 arrayData[:,i,:] = data[blockList,:]
454 arrayData[:,i,:] = data[blockList,:]
455
455
456 # Three dimensions
456 # Three dimensions
457 else:
457 else:
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 for i in range(nDatas):
459 for i in range(nDatas):
460
460
461 data = dataset[strds + str(i)].value
461 data = dataset[strds + str(i)].value
462
462
463 for b in range(blockList.size):
463 for b in range(blockList.size):
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465
465
466 return arrayData
466 return arrayData
467
467
468 def __setDataOut(self):
468 def __setDataOut(self):
469 listMeta = self.listMeta
469 listMeta = self.listMeta
470 listMetaname = self.listMetaname
470 listMetaname = self.listMetaname
471 listDataname = self.listDataname
471 listDataname = self.listDataname
472 listData = self.listData
472 listData = self.listData
473 listShapes = self.listShapes
473 listShapes = self.listShapes
474
474
475 blockIndex = self.blockIndex
475 blockIndex = self.blockIndex
476 # blockList = self.blockList
476 # blockList = self.blockList
477
477
478 for i in range(len(listMeta)):
478 for i in range(len(listMeta)):
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
480
480
481 for j in range(len(listData)):
481 for j in range(len(listData)):
482 nShapes = listShapes[listDataname[j]][0]
482 nShapes = listShapes[listDataname[j]][0]
483 mode = listShapes[listDataname[j]][4]
483 mode = listShapes[listDataname[j]][4]
484 if nShapes == 1:
484 if nShapes == 1:
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 elif nShapes > 1:
486 elif nShapes > 1:
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 elif mode==0:
488 elif mode==0:
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 #Mode Meteors
490 #Mode Meteors
491 elif mode ==2:
491 elif mode ==2:
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 setattr(self.dataOut, listDataname[j], selectedData)
493 setattr(self.dataOut, listDataname[j], selectedData)
494 return
494 return
495
495
496 def __selectDataMode2(self, data, blockIndex):
496 def __selectDataMode2(self, data, blockIndex):
497 utctime = data[:,0]
497 utctime = data[:,0]
498 aux, indices = numpy.unique(utctime, return_inverse=True)
498 aux, indices = numpy.unique(utctime, return_inverse=True)
499 selInd = numpy.where(indices == blockIndex)[0]
499 selInd = numpy.where(indices == blockIndex)[0]
500 selData = data[selInd,:]
500 selData = data[selInd,:]
501
501
502 return selData
502 return selData
503
503
504 def getData(self):
504 def getData(self):
505
505
506 # if self.flagNoMoreFiles:
506 # if self.flagNoMoreFiles:
507 # self.dataOut.flagNoData = True
507 # self.dataOut.flagNoData = True
508 # print 'Process finished'
508 # print 'Process finished'
509 # return 0
509 # return 0
510 #
510 #
511 if self.blockIndex==self.blocksPerFile:
511 if self.blockIndex==self.blocksPerFile:
512 if not( self.__setNextFileOffline() ):
512 if not( self.__setNextFileOffline() ):
513 self.dataOut.flagNoData = True
513 self.dataOut.flagNoData = True
514 return 0
514 return 0
515
515
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 # self.dataOut.flagNoData = True
517 # self.dataOut.flagNoData = True
518 # return 0
518 # return 0
519 # self.__readData()
519 # self.__readData()
520 self.__setDataOut()
520 self.__setDataOut()
521 self.dataOut.flagNoData = False
521 self.dataOut.flagNoData = False
522
522
523 self.blockIndex += 1
523 self.blockIndex += 1
524
524
525 return
525 return
526
526
527 def run(self, **kwargs):
527 def run(self, **kwargs):
528
528
529 if not(self.isConfig):
529 if not(self.isConfig):
530 self.setup(**kwargs)
530 self.setup(**kwargs)
531 # self.setObjProperties()
531 # self.setObjProperties()
532 self.isConfig = True
532 self.isConfig = True
533
533
534 self.getData()
534 self.getData()
535
535
536 return
536 return
537
537
538 class ParamWriter(Operation):
538 class ParamWriter(Operation):
539 '''
539 '''
540 HDF5 Writer, stores parameters data in HDF5 format files
540 HDF5 Writer, stores parameters data in HDF5 format files
541
541
542 path: path where the files will be stored
542 path: path where the files will be stored
543
543
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545
545
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547
547
548 metadataList: list of attributes that will be stored as metadata
548 metadataList: list of attributes that will be stored as metadata
549
549
550 dataList: list of attributes that will be stores as data
550 dataList: list of attributes that will be stores as data
551
551
552 '''
552 '''
553
553
554
554
555 ext = ".hdf5"
555 ext = ".hdf5"
556
556
557 optchar = "D"
557 optchar = "D"
558
558
559 metaoptchar = "M"
559 metaoptchar = "M"
560
560
561 metaFile = None
561 metaFile = None
562
562
563 filename = None
563 filename = None
564
564
565 path = None
565 path = None
566
566
567 setFile = None
567 setFile = None
568
568
569 fp = None
569 fp = None
570
570
571 grp = None
571 grp = None
572
572
573 ds = None
573 ds = None
574
574
575 firsttime = True
575 firsttime = True
576
576
577 #Configurations
577 #Configurations
578
578
579 blocksPerFile = None
579 blocksPerFile = None
580
580
581 blockIndex = None
581 blockIndex = None
582
582
583 dataOut = None
583 dataOut = None
584
584
585 #Data Arrays
585 #Data Arrays
586
586
587 dataList = None
587 dataList = None
588
588
589 metadataList = None
589 metadataList = None
590
590
591 # arrayDim = None
591 # arrayDim = None
592
592
593 dsList = None #List of dictionaries with dataset properties
593 dsList = None #List of dictionaries with dataset properties
594
594
595 tableDim = None
595 tableDim = None
596
596
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598
598
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600
600
601 currentDay = None
601 currentDay = None
602
602
603 lastTime = None
603 lastTime = None
604
604
605 def __init__(self, **kwargs):
605 def __init__(self, **kwargs):
606 Operation.__init__(self, **kwargs)
606 Operation.__init__(self, **kwargs)
607 self.isConfig = False
607 self.isConfig = False
608 return
608 return
609
609
610 def setup(self, dataOut, **kwargs):
610 def setup(self, dataOut, **kwargs):
611
611
612 self.path = kwargs['path']
612 self.path = kwargs['path']
613 self.setType = kwargs.get('setType', None)
613
614
614 if kwargs.has_key('blocksPerFile'):
615 if kwargs.has_key('blocksPerFile'):
615 self.blocksPerFile = kwargs['blocksPerFile']
616 self.blocksPerFile = kwargs['blocksPerFile']
616 else:
617 else:
617 self.blocksPerFile = 10
618 self.blocksPerFile = 10
618
619
619 self.metadataList = kwargs['metadataList']
620 self.metadataList = kwargs['metadataList']
620 self.dataList = kwargs['dataList']
621 self.dataList = kwargs['dataList']
621 self.dataOut = dataOut
622 self.dataOut = dataOut
622
623
623 if kwargs.has_key('mode'):
624 if kwargs.has_key('mode'):
624 mode = kwargs['mode']
625 mode = kwargs['mode']
625
626
626 if type(mode) == int:
627 if type(mode) == int:
627 mode = numpy.zeros(len(self.dataList)) + mode
628 mode = numpy.zeros(len(self.dataList)) + mode
628 else:
629 else:
629 mode = numpy.ones(len(self.dataList))
630 mode = numpy.ones(len(self.dataList))
630
631
631 self.mode = mode
632 self.mode = mode
632
633
633 arrayDim = numpy.zeros((len(self.dataList),5))
634 arrayDim = numpy.zeros((len(self.dataList),5))
634
635
635 #Table dimensions
636 #Table dimensions
636 dtype0 = self.dtype
637 dtype0 = self.dtype
637 tableList = []
638 tableList = []
638
639
639 #Dictionary and list of tables
640 #Dictionary and list of tables
640 dsList = []
641 dsList = []
641
642
642 for i in range(len(self.dataList)):
643 for i in range(len(self.dataList)):
643 dsDict = {}
644 dsDict = {}
644 dataAux = getattr(self.dataOut, self.dataList[i])
645 dataAux = getattr(self.dataOut, self.dataList[i])
645 dsDict['variable'] = self.dataList[i]
646 dsDict['variable'] = self.dataList[i]
646 #--------------------- Conditionals ------------------------
647 #--------------------- Conditionals ------------------------
647 #There is no data
648 #There is no data
648 if dataAux is None:
649 if dataAux is None:
649 return 0
650 return 0
650
651
651 #Not array, just a number
652 #Not array, just a number
652 #Mode 0
653 #Mode 0
653 if type(dataAux)==float or type(dataAux)==int:
654 if type(dataAux)==float or type(dataAux)==int:
654 dsDict['mode'] = 0
655 dsDict['mode'] = 0
655 dsDict['nDim'] = 0
656 dsDict['nDim'] = 0
656 arrayDim[i,0] = 0
657 arrayDim[i,0] = 0
657 dsList.append(dsDict)
658 dsList.append(dsDict)
658
659
659 #Mode 2: meteors
660 #Mode 2: meteors
660 elif mode[i] == 2:
661 elif mode[i] == 2:
661 # dsDict['nDim'] = 0
662 # dsDict['nDim'] = 0
662 dsDict['dsName'] = 'table0'
663 dsDict['dsName'] = 'table0'
663 dsDict['mode'] = 2 # Mode meteors
664 dsDict['mode'] = 2 # Mode meteors
664 dsDict['shape'] = dataAux.shape[-1]
665 dsDict['shape'] = dataAux.shape[-1]
665 dsDict['nDim'] = 0
666 dsDict['nDim'] = 0
666 dsDict['dsNumber'] = 1
667 dsDict['dsNumber'] = 1
667
668
668 arrayDim[i,3] = dataAux.shape[-1]
669 arrayDim[i,3] = dataAux.shape[-1]
669 arrayDim[i,4] = mode[i] #Mode the data was stored
670 arrayDim[i,4] = mode[i] #Mode the data was stored
670
671
671 dsList.append(dsDict)
672 dsList.append(dsDict)
672
673
673 #Mode 1
674 #Mode 1
674 else:
675 else:
675 arrayDim0 = dataAux.shape #Data dimensions
676 arrayDim0 = dataAux.shape #Data dimensions
676 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
677 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
677 arrayDim[i,4] = mode[i] #Mode the data was stored
678 arrayDim[i,4] = mode[i] #Mode the data was stored
678
679
679 strtable = 'table'
680 strtable = 'table'
680 dsDict['mode'] = 1 # Mode parameters
681 dsDict['mode'] = 1 # Mode parameters
681
682
682 # Three-dimension arrays
683 # Three-dimension arrays
683 if len(arrayDim0) == 3:
684 if len(arrayDim0) == 3:
684 arrayDim[i,1:-1] = numpy.array(arrayDim0)
685 arrayDim[i,1:-1] = numpy.array(arrayDim0)
685 nTables = int(arrayDim[i,2])
686 nTables = int(arrayDim[i,2])
686 dsDict['dsNumber'] = nTables
687 dsDict['dsNumber'] = nTables
687 dsDict['shape'] = arrayDim[i,2:4]
688 dsDict['shape'] = arrayDim[i,2:4]
688 dsDict['nDim'] = 3
689 dsDict['nDim'] = 3
689
690
690 for j in range(nTables):
691 for j in range(nTables):
691 dsDict = dsDict.copy()
692 dsDict = dsDict.copy()
692 dsDict['dsName'] = strtable + str(j)
693 dsDict['dsName'] = strtable + str(j)
693 dsList.append(dsDict)
694 dsList.append(dsDict)
694
695
695 # Two-dimension arrays
696 # Two-dimension arrays
696 elif len(arrayDim0) == 2:
697 elif len(arrayDim0) == 2:
697 arrayDim[i,2:-1] = numpy.array(arrayDim0)
698 arrayDim[i,2:-1] = numpy.array(arrayDim0)
698 nTables = int(arrayDim[i,2])
699 nTables = int(arrayDim[i,2])
699 dsDict['dsNumber'] = nTables
700 dsDict['dsNumber'] = nTables
700 dsDict['shape'] = arrayDim[i,3]
701 dsDict['shape'] = arrayDim[i,3]
701 dsDict['nDim'] = 2
702 dsDict['nDim'] = 2
702
703
703 for j in range(nTables):
704 for j in range(nTables):
704 dsDict = dsDict.copy()
705 dsDict = dsDict.copy()
705 dsDict['dsName'] = strtable + str(j)
706 dsDict['dsName'] = strtable + str(j)
706 dsList.append(dsDict)
707 dsList.append(dsDict)
707
708
708 # One-dimension arrays
709 # One-dimension arrays
709 elif len(arrayDim0) == 1:
710 elif len(arrayDim0) == 1:
710 arrayDim[i,3] = arrayDim0[0]
711 arrayDim[i,3] = arrayDim0[0]
711 dsDict['shape'] = arrayDim0[0]
712 dsDict['shape'] = arrayDim0[0]
712 dsDict['dsNumber'] = 1
713 dsDict['dsNumber'] = 1
713 dsDict['dsName'] = strtable + str(0)
714 dsDict['dsName'] = strtable + str(0)
714 dsDict['nDim'] = 1
715 dsDict['nDim'] = 1
715 dsList.append(dsDict)
716 dsList.append(dsDict)
716
717
717 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
718 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
718 tableList.append(table)
719 tableList.append(table)
719
720
720 # self.arrayDim = arrayDim
721 # self.arrayDim = arrayDim
721 self.dsList = dsList
722 self.dsList = dsList
722 self.tableDim = numpy.array(tableList, dtype = dtype0)
723 self.tableDim = numpy.array(tableList, dtype = dtype0)
723 self.blockIndex = 0
724 self.blockIndex = 0
724
725
725 timeTuple = time.localtime(dataOut.utctime)
726 timeTuple = time.localtime(dataOut.utctime)
726 self.currentDay = timeTuple.tm_yday
727 self.currentDay = timeTuple.tm_yday
727 return 1
728 return 1
728
729
729 def putMetadata(self):
730 def putMetadata(self):
730
731
731 fp = self.createMetadataFile()
732 fp = self.createMetadataFile()
732 self.writeMetadata(fp)
733 self.writeMetadata(fp)
733 fp.close()
734 fp.close()
734 return
735 return
735
736
736 def createMetadataFile(self):
737 def createMetadataFile(self):
737 ext = self.ext
738 ext = self.ext
738 path = self.path
739 path = self.path
739 setFile = self.setFile
740 setFile = self.setFile
740
741
741 timeTuple = time.localtime(self.dataOut.utctime)
742 timeTuple = time.localtime(self.dataOut.utctime)
742
743
743 subfolder = ''
744 subfolder = ''
744 fullpath = os.path.join( path, subfolder )
745 fullpath = os.path.join( path, subfolder )
745
746
746 if not( os.path.exists(fullpath) ):
747 if not( os.path.exists(fullpath) ):
747 os.mkdir(fullpath)
748 os.mkdir(fullpath)
748 setFile = -1 #inicializo mi contador de seteo
749 setFile = -1 #inicializo mi contador de seteo
749
750
750 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
751 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
751 fullpath = os.path.join( path, subfolder )
752 fullpath = os.path.join( path, subfolder )
752
753
753 if not( os.path.exists(fullpath) ):
754 if not( os.path.exists(fullpath) ):
754 os.mkdir(fullpath)
755 os.mkdir(fullpath)
755 setFile = -1 #inicializo mi contador de seteo
756 setFile = -1 #inicializo mi contador de seteo
756
757
757 else:
758 else:
758 filesList = os.listdir( fullpath )
759 filesList = os.listdir( fullpath )
759 filesList = sorted( filesList, key=str.lower )
760 filesList = sorted( filesList, key=str.lower )
760 if len( filesList ) > 0:
761 if len( filesList ) > 0:
761 filesList = [k for k in filesList if 'M' in k]
762 filesList = [k for k in filesList if 'M' in k]
762 filen = filesList[-1]
763 filen = filesList[-1]
763 # el filename debera tener el siguiente formato
764 # el filename debera tener el siguiente formato
764 # 0 1234 567 89A BCDE (hex)
765 # 0 1234 567 89A BCDE (hex)
765 # x YYYY DDD SSS .ext
766 # x YYYY DDD SSS .ext
766 if isNumber( filen[8:11] ):
767 if isNumber( filen[8:11] ):
767 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
768 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
768 else:
769 else:
769 setFile = -1
770 setFile = -1
770 else:
771 else:
771 setFile = -1 #inicializo mi contador de seteo
772 setFile = -1 #inicializo mi contador de seteo
772
773
773 setFile += 1
774 if self.setType is None:
774
775 setFile += 1
775 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
776 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
776 timeTuple.tm_year,
777 timeTuple.tm_year,
777 timeTuple.tm_yday,
778 timeTuple.tm_yday,
778 setFile,
779 setFile,
779 ext )
780 ext )
781 else:
782 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
783 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
784 timeTuple.tm_year,
785 timeTuple.tm_yday,
786 setFile,
787 ext )
780
788
781 filename = os.path.join( path, subfolder, file )
789 filename = os.path.join( path, subfolder, file )
782 self.metaFile = file
790 self.metaFile = file
783 #Setting HDF5 File
791 #Setting HDF5 File
784 fp = h5py.File(filename,'w')
792 fp = h5py.File(filename,'w')
785
793
786 return fp
794 return fp
787
795
788 def writeMetadata(self, fp):
796 def writeMetadata(self, fp):
789
797
790 grp = fp.create_group("Metadata")
798 grp = fp.create_group("Metadata")
791 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
799 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
792
800
793 for i in range(len(self.metadataList)):
801 for i in range(len(self.metadataList)):
794 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
802 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
795 return
803 return
796
804
797 def timeFlag(self):
805 def timeFlag(self):
798 currentTime = self.dataOut.utctime
806 currentTime = self.dataOut.utctime
799
807
800 if self.lastTime is None:
808 if self.lastTime is None:
801 self.lastTime = currentTime
809 self.lastTime = currentTime
802
810
803 #Day
811 #Day
804 timeTuple = time.localtime(currentTime)
812 timeTuple = time.localtime(currentTime)
805 dataDay = timeTuple.tm_yday
813 dataDay = timeTuple.tm_yday
806
814
807 #Time
815 #Time
808 timeDiff = currentTime - self.lastTime
816 timeDiff = currentTime - self.lastTime
809
817
810 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
818 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
811 if dataDay != self.currentDay:
819 if dataDay != self.currentDay:
812 self.currentDay = dataDay
820 self.currentDay = dataDay
813 return True
821 return True
814 elif timeDiff > 3*60*60:
822 elif timeDiff > 3*60*60:
815 self.lastTime = currentTime
823 self.lastTime = currentTime
816 return True
824 return True
817 else:
825 else:
818 self.lastTime = currentTime
826 self.lastTime = currentTime
819 return False
827 return False
820
828
821 def setNextFile(self):
829 def setNextFile(self):
822
830
823 ext = self.ext
831 ext = self.ext
824 path = self.path
832 path = self.path
825 setFile = self.setFile
833 setFile = self.setFile
826 mode = self.mode
834 mode = self.mode
827
835
828 timeTuple = time.localtime(self.dataOut.utctime)
836 timeTuple = time.localtime(self.dataOut.utctime)
829 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
837 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
830
838
831 fullpath = os.path.join( path, subfolder )
839 fullpath = os.path.join( path, subfolder )
832
840
833 if os.path.exists(fullpath):
841 if os.path.exists(fullpath):
834 filesList = os.listdir( fullpath )
842 filesList = os.listdir( fullpath )
835 filesList = [k for k in filesList if 'D' in k]
843 filesList = [k for k in filesList if 'D' in k]
836 if len( filesList ) > 0:
844 if len( filesList ) > 0:
837 filesList = sorted( filesList, key=str.lower )
845 filesList = sorted( filesList, key=str.lower )
838 filen = filesList[-1]
846 filen = filesList[-1]
839 # el filename debera tener el siguiente formato
847 # el filename debera tener el siguiente formato
840 # 0 1234 567 89A BCDE (hex)
848 # 0 1234 567 89A BCDE (hex)
841 # x YYYY DDD SSS .ext
849 # x YYYY DDD SSS .ext
842 if isNumber( filen[8:11] ):
850 if isNumber( filen[8:11] ):
843 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
851 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
844 else:
852 else:
845 setFile = -1
853 setFile = -1
846 else:
854 else:
847 setFile = -1 #inicializo mi contador de seteo
855 setFile = -1 #inicializo mi contador de seteo
848 else:
856 else:
849 os.makedirs(fullpath)
857 os.makedirs(fullpath)
850 setFile = -1 #inicializo mi contador de seteo
858 setFile = -1 #inicializo mi contador de seteo
851
859
852 setFile += 1
860 if self.setType is None:
853
861 setFile += 1
854 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
862 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
855 timeTuple.tm_year,
863 timeTuple.tm_year,
856 timeTuple.tm_yday,
864 timeTuple.tm_yday,
857 setFile,
865 setFile,
858 ext )
866 ext )
867 else:
868 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
869 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
870 timeTuple.tm_year,
871 timeTuple.tm_yday,
872 setFile,
873 ext )
859
874
860 filename = os.path.join( path, subfolder, file )
875 filename = os.path.join( path, subfolder, file )
861
876
862 #Setting HDF5 File
877 #Setting HDF5 File
863 fp = h5py.File(filename,'w')
878 fp = h5py.File(filename,'w')
864 #write metadata
879 #write metadata
865 self.writeMetadata(fp)
880 self.writeMetadata(fp)
866 #Write data
881 #Write data
867 grp = fp.create_group("Data")
882 grp = fp.create_group("Data")
868 # grp.attrs['metadata'] = self.metaFile
883 # grp.attrs['metadata'] = self.metaFile
869
884
870 # grp.attrs['blocksPerFile'] = 0
885 # grp.attrs['blocksPerFile'] = 0
871 ds = []
886 ds = []
872 data = []
887 data = []
873 dsList = self.dsList
888 dsList = self.dsList
874 i = 0
889 i = 0
875 while i < len(dsList):
890 while i < len(dsList):
876 dsInfo = dsList[i]
891 dsInfo = dsList[i]
877 #One-dimension data
892 #One-dimension data
878 if dsInfo['mode'] == 0:
893 if dsInfo['mode'] == 0:
879 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
894 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
880 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
895 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
881 ds.append(ds0)
896 ds.append(ds0)
882 data.append([])
897 data.append([])
883 i += 1
898 i += 1
884 continue
899 continue
885 # nDimsForDs.append(nDims[i])
900 # nDimsForDs.append(nDims[i])
886
901
887 elif dsInfo['mode'] == 2:
902 elif dsInfo['mode'] == 2:
888 grp0 = grp.create_group(dsInfo['variable'])
903 grp0 = grp.create_group(dsInfo['variable'])
889 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
904 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
890 ds.append(ds0)
905 ds.append(ds0)
891 data.append([])
906 data.append([])
892 i += 1
907 i += 1
893 continue
908 continue
894
909
895 elif dsInfo['mode'] == 1:
910 elif dsInfo['mode'] == 1:
896 grp0 = grp.create_group(dsInfo['variable'])
911 grp0 = grp.create_group(dsInfo['variable'])
897
912
898 for j in range(dsInfo['dsNumber']):
913 for j in range(dsInfo['dsNumber']):
899 dsInfo = dsList[i]
914 dsInfo = dsList[i]
900 tableName = dsInfo['dsName']
915 tableName = dsInfo['dsName']
901 shape = int(dsInfo['shape'])
916 shape = int(dsInfo['shape'])
902
917
903 if dsInfo['nDim'] == 3:
918 if dsInfo['nDim'] == 3:
904 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
919 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
905 else:
920 else:
906 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
921 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
907
922
908 ds.append(ds0)
923 ds.append(ds0)
909 data.append([])
924 data.append([])
910 i += 1
925 i += 1
911 # nDimsForDs.append(nDims[i])
926 # nDimsForDs.append(nDims[i])
912
927
913 fp.flush()
928 fp.flush()
914 fp.close()
929 fp.close()
915
930
916 # self.nDatas = nDatas
931 # self.nDatas = nDatas
917 # self.nDims = nDims
932 # self.nDims = nDims
918 # self.nDimsForDs = nDimsForDs
933 # self.nDimsForDs = nDimsForDs
919 #Saving variables
934 #Saving variables
920 print 'Writing the file: %s'%filename
935 print 'Writing the file: %s'%filename
921 self.filename = filename
936 self.filename = filename
922 # self.fp = fp
937 # self.fp = fp
923 # self.grp = grp
938 # self.grp = grp
924 # self.grp.attrs.modify('nRecords', 1)
939 # self.grp.attrs.modify('nRecords', 1)
925 self.ds = ds
940 self.ds = ds
926 self.data = data
941 self.data = data
927 # self.setFile = setFile
942 # self.setFile = setFile
928 self.firsttime = True
943 self.firsttime = True
929 self.blockIndex = 0
944 self.blockIndex = 0
930 return
945 return
931
946
932 def putData(self):
947 def putData(self):
933
948
934 if self.blockIndex == self.blocksPerFile or self.timeFlag():
949 if self.blockIndex == self.blocksPerFile or self.timeFlag():
935 self.setNextFile()
950 self.setNextFile()
936
951
937 # if not self.firsttime:
952 # if not self.firsttime:
938 self.readBlock()
953 self.readBlock()
939 self.setBlock() #Prepare data to be written
954 self.setBlock() #Prepare data to be written
940 self.writeBlock() #Write data
955 self.writeBlock() #Write data
941
956
942 return
957 return
943
958
944 def readBlock(self):
959 def readBlock(self):
945
960
946 '''
961 '''
947 data Array configured
962 data Array configured
948
963
949
964
950 self.data
965 self.data
951 '''
966 '''
952 dsList = self.dsList
967 dsList = self.dsList
953 ds = self.ds
968 ds = self.ds
954 #Setting HDF5 File
969 #Setting HDF5 File
955 fp = h5py.File(self.filename,'r+')
970 fp = h5py.File(self.filename,'r+')
956 grp = fp["Data"]
971 grp = fp["Data"]
957 ind = 0
972 ind = 0
958
973
959 # grp.attrs['blocksPerFile'] = 0
974 # grp.attrs['blocksPerFile'] = 0
960 while ind < len(dsList):
975 while ind < len(dsList):
961 dsInfo = dsList[ind]
976 dsInfo = dsList[ind]
962
977
963 if dsInfo['mode'] == 0:
978 if dsInfo['mode'] == 0:
964 ds0 = grp[dsInfo['variable']]
979 ds0 = grp[dsInfo['variable']]
965 ds[ind] = ds0
980 ds[ind] = ds0
966 ind += 1
981 ind += 1
967 else:
982 else:
968
983
969 grp0 = grp[dsInfo['variable']]
984 grp0 = grp[dsInfo['variable']]
970
985
971 for j in range(dsInfo['dsNumber']):
986 for j in range(dsInfo['dsNumber']):
972 dsInfo = dsList[ind]
987 dsInfo = dsList[ind]
973 ds0 = grp0[dsInfo['dsName']]
988 ds0 = grp0[dsInfo['dsName']]
974 ds[ind] = ds0
989 ds[ind] = ds0
975 ind += 1
990 ind += 1
976
991
977 self.fp = fp
992 self.fp = fp
978 self.grp = grp
993 self.grp = grp
979 self.ds = ds
994 self.ds = ds
980
995
981 return
996 return
982
997
983 def setBlock(self):
998 def setBlock(self):
984 '''
999 '''
985 data Array configured
1000 data Array configured
986
1001
987
1002
988 self.data
1003 self.data
989 '''
1004 '''
990 #Creating Arrays
1005 #Creating Arrays
991 dsList = self.dsList
1006 dsList = self.dsList
992 data = self.data
1007 data = self.data
993 ind = 0
1008 ind = 0
994
1009
995 while ind < len(dsList):
1010 while ind < len(dsList):
996 dsInfo = dsList[ind]
1011 dsInfo = dsList[ind]
997 dataAux = getattr(self.dataOut, dsInfo['variable'])
1012 dataAux = getattr(self.dataOut, dsInfo['variable'])
998
1013
999 mode = dsInfo['mode']
1014 mode = dsInfo['mode']
1000 nDim = dsInfo['nDim']
1015 nDim = dsInfo['nDim']
1001
1016
1002 if mode == 0 or mode == 2 or nDim == 1:
1017 if mode == 0 or mode == 2 or nDim == 1:
1003 data[ind] = dataAux
1018 data[ind] = dataAux
1004 ind += 1
1019 ind += 1
1005 # elif nDim == 1:
1020 # elif nDim == 1:
1006 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1021 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1007 # ind += 1
1022 # ind += 1
1008 elif nDim == 2:
1023 elif nDim == 2:
1009 for j in range(dsInfo['dsNumber']):
1024 for j in range(dsInfo['dsNumber']):
1010 data[ind] = dataAux[j,:]
1025 data[ind] = dataAux[j,:]
1011 ind += 1
1026 ind += 1
1012 elif nDim == 3:
1027 elif nDim == 3:
1013 for j in range(dsInfo['dsNumber']):
1028 for j in range(dsInfo['dsNumber']):
1014 data[ind] = dataAux[:,j,:]
1029 data[ind] = dataAux[:,j,:]
1015 ind += 1
1030 ind += 1
1016
1031
1017 self.data = data
1032 self.data = data
1018 return
1033 return
1019
1034
1020 def writeBlock(self):
1035 def writeBlock(self):
1021 '''
1036 '''
1022 Saves the block in the HDF5 file
1037 Saves the block in the HDF5 file
1023 '''
1038 '''
1024 dsList = self.dsList
1039 dsList = self.dsList
1025
1040
1026 for i in range(len(self.ds)):
1041 for i in range(len(self.ds)):
1027 dsInfo = dsList[i]
1042 dsInfo = dsList[i]
1028 nDim = dsInfo['nDim']
1043 nDim = dsInfo['nDim']
1029 mode = dsInfo['mode']
1044 mode = dsInfo['mode']
1030
1045
1031 # First time
1046 # First time
1032 if self.firsttime:
1047 if self.firsttime:
1033 # self.ds[i].resize(self.data[i].shape)
1048 # self.ds[i].resize(self.data[i].shape)
1034 # self.ds[i][self.blockIndex,:] = self.data[i]
1049 # self.ds[i][self.blockIndex,:] = self.data[i]
1035 if type(self.data[i]) == numpy.ndarray:
1050 if type(self.data[i]) == numpy.ndarray:
1036
1051
1037 if nDim == 3:
1052 if nDim == 3:
1038 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1053 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1039 self.ds[i].resize(self.data[i].shape)
1054 self.ds[i].resize(self.data[i].shape)
1040 if mode == 2:
1055 if mode == 2:
1041 self.ds[i].resize(self.data[i].shape)
1056 self.ds[i].resize(self.data[i].shape)
1042 self.ds[i][:] = self.data[i]
1057 self.ds[i][:] = self.data[i]
1043 else:
1058 else:
1044
1059
1045 # From second time
1060 # From second time
1046 # Meteors!
1061 # Meteors!
1047 if mode == 2:
1062 if mode == 2:
1048 dataShape = self.data[i].shape
1063 dataShape = self.data[i].shape
1049 dsShape = self.ds[i].shape
1064 dsShape = self.ds[i].shape
1050 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1065 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1051 self.ds[i][dsShape[0]:,:] = self.data[i]
1066 self.ds[i][dsShape[0]:,:] = self.data[i]
1052 # No dimension
1067 # No dimension
1053 elif mode == 0:
1068 elif mode == 0:
1054 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1069 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1055 self.ds[i][0,-1] = self.data[i]
1070 self.ds[i][0,-1] = self.data[i]
1056 # One dimension
1071 # One dimension
1057 elif nDim == 1:
1072 elif nDim == 1:
1058 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1073 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1059 self.ds[i][-1,:] = self.data[i]
1074 self.ds[i][-1,:] = self.data[i]
1060 # Two dimension
1075 # Two dimension
1061 elif nDim == 2:
1076 elif nDim == 2:
1062 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1077 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1063 self.ds[i][self.blockIndex,:] = self.data[i]
1078 self.ds[i][self.blockIndex,:] = self.data[i]
1064 # Three dimensions
1079 # Three dimensions
1065 elif nDim == 3:
1080 elif nDim == 3:
1066 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1081 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1067 self.ds[i][:,:,-1] = self.data[i]
1082 self.ds[i][:,:,-1] = self.data[i]
1068
1083
1069 self.firsttime = False
1084 self.firsttime = False
1070 self.blockIndex += 1
1085 self.blockIndex += 1
1071
1086
1072 #Close to save changes
1087 #Close to save changes
1073 self.fp.flush()
1088 self.fp.flush()
1074 self.fp.close()
1089 self.fp.close()
1075 return
1090 return
1076
1091
1077 def run(self, dataOut, **kwargs):
1092 def run(self, dataOut, **kwargs):
1078
1093
1079 if not(self.isConfig):
1094 if not(self.isConfig):
1080 flagdata = self.setup(dataOut, **kwargs)
1095 flagdata = self.setup(dataOut, **kwargs)
1081
1096
1082 if not(flagdata):
1097 if not(flagdata):
1083 return
1098 return
1084
1099
1085 self.isConfig = True
1100 self.isConfig = True
1086 # self.putMetadata()
1101 # self.putMetadata()
1087 self.setNextFile()
1102 self.setNextFile()
1088
1103
1089 self.putData()
1104 self.putData()
1090 return
1105 return
General Comments 0
You need to be logged in to leave comments. Login now