##// END OF EJS Templates
schain-jc
RSS
Test commit jroIO_param.py change letter D by M
soporte -
r1216:cef0eac4db4e v2.3-devel
parent child
Show More
Side by Side Unified
Context file:
r1216:cef0eac4db4e Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1095 +1,1095
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class ParamReader(ProcessingUnit):
15 class ParamReader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self, **kwargs):
77 def __init__(self, **kwargs):
78 ProcessingUnit.__init__(self, **kwargs)
78 ProcessingUnit.__init__(self, **kwargs)
79 self.dataOut = Parameters()
79 self.dataOut = Parameters()
80 return
80 return
81
81
82 def setup(self, **kwargs):
82 def setup(self, **kwargs):
83
83
84 path = kwargs['path']
84 path = kwargs['path']
85 startDate = kwargs['startDate']
85 startDate = kwargs['startDate']
86 endDate = kwargs['endDate']
86 endDate = kwargs['endDate']
87 startTime = kwargs['startTime']
87 startTime = kwargs['startTime']
88 endTime = kwargs['endTime']
88 endTime = kwargs['endTime']
89 walk = kwargs['walk']
89 walk = kwargs['walk']
90 if kwargs.has_key('ext'):
90 if kwargs.has_key('ext'):
91 ext = kwargs['ext']
91 ext = kwargs['ext']
92 else:
92 else:
93 ext = '.hdf5'
93 ext = '.hdf5'
94 if kwargs.has_key('timezone'):
94 if kwargs.has_key('timezone'):
95 self.timezone = kwargs['timezone']
95 self.timezone = kwargs['timezone']
96 else:
96 else:
97 self.timezone = 'lt'
97 self.timezone = 'lt'
98
98
99 print "[Reading] Searching files in offline mode ..."
99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 startTime=startTime, endTime=endTime,
101 startTime=startTime, endTime=endTime,
102 ext=ext, walk=walk)
102 ext=ext, walk=walk)
103
103
104 if not(filenameList):
104 if not(filenameList):
105 print "There is no files into the folder: %s"%(path)
105 print "There is no files into the folder: %s"%(path)
106 sys.exit(-1)
106 sys.exit(-1)
107
107
108 self.fileIndex = -1
108 self.fileIndex = -1
109 self.startTime = startTime
109 self.startTime = startTime
110 self.endTime = endTime
110 self.endTime = endTime
111
111
112 self.__readMetadata()
112 self.__readMetadata()
113
113
114 self.__setNextFileOffline()
114 self.__setNextFileOffline()
115
115
116 return
116 return
117
117
118 def searchFilesOffLine(self,
118 def searchFilesOffLine(self,
119 path,
119 path,
120 startDate=None,
120 startDate=None,
121 endDate=None,
121 endDate=None,
122 startTime=datetime.time(0,0,0),
122 startTime=datetime.time(0,0,0),
123 endTime=datetime.time(23,59,59),
123 endTime=datetime.time(23,59,59),
124 ext='.hdf5',
124 ext='.hdf5',
125 walk=True):
125 walk=True):
126
126
127 expLabel = ''
127 expLabel = ''
128 self.filenameList = []
128 self.filenameList = []
129 self.datetimeList = []
129 self.datetimeList = []
130
130
131 pathList = []
131 pathList = []
132
132
133 JRODataObj = JRODataReader()
133 JRODataObj = JRODataReader()
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135
135
136 if dateList == []:
136 if dateList == []:
137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 datetime.datetime.combine(startDate,startTime).ctime(),
138 datetime.datetime.combine(startDate,startTime).ctime(),
139 datetime.datetime.combine(endDate,endTime).ctime())
139 datetime.datetime.combine(endDate,endTime).ctime())
140
140
141 return None, None
141 return None, None
142
142
143 if len(dateList) > 1:
143 if len(dateList) > 1:
144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
145 else:
145 else:
146 print "[Reading] data was found for the date %s" %(dateList[0])
146 print "[Reading] data was found for the date %s" %(dateList[0])
147
147
148 filenameList = []
148 filenameList = []
149 datetimeList = []
149 datetimeList = []
150
150
151 #----------------------------------------------------------------------------------
151 #----------------------------------------------------------------------------------
152
152
153 for thisPath in pathList:
153 for thisPath in pathList:
154 # thisPath = pathList[pathDict[file]]
154 # thisPath = pathList[pathDict[file]]
155
155
156 fileList = glob.glob1(thisPath, "*%s" %ext)
156 fileList = glob.glob1(thisPath, "*%s" %ext)
157 fileList.sort()
157 fileList.sort()
158
158
159 for file in fileList:
159 for file in fileList:
160
160
161 filename = os.path.join(thisPath,file)
161 filename = os.path.join(thisPath,file)
162
162
163 if not isFileInDateRange(filename, startDate, endDate):
163 if not isFileInDateRange(filename, startDate, endDate):
164 continue
164 continue
165
165
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167
167
168 if not(thisDatetime):
168 if not(thisDatetime):
169 continue
169 continue
170
170
171 filenameList.append(filename)
171 filenameList.append(filename)
172 datetimeList.append(thisDatetime)
172 datetimeList.append(thisDatetime)
173
173
174 if not(filenameList):
174 if not(filenameList):
175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
176 return None, None
176 return None, None
177
177
178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
179 print
179 print
180
180
181 # for i in range(len(filenameList)):
181 # for i in range(len(filenameList)):
182 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
182 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183
183
184 self.filenameList = filenameList
184 self.filenameList = filenameList
185 self.datetimeList = datetimeList
185 self.datetimeList = datetimeList
186
186
187 return pathList, filenameList
187 return pathList, filenameList
188
188
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190
190
191 """
191 """
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193
193
194 Inputs:
194 Inputs:
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196
196
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
198
198
199 endDate : fecha final del rango seleccionado en formato datetime.date
199 endDate : fecha final del rango seleccionado en formato datetime.date
200
200
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202
202
203 endTime : tiempo final del rango seleccionado en formato datetime.time
203 endTime : tiempo final del rango seleccionado en formato datetime.time
204
204
205 Return:
205 Return:
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 fecha especificado, de lo contrario retorna False.
207 fecha especificado, de lo contrario retorna False.
208
208
209 Excepciones:
209 Excepciones:
210 Si el archivo no existe o no puede ser abierto
210 Si el archivo no existe o no puede ser abierto
211 Si la cabecera no puede ser leida.
211 Si la cabecera no puede ser leida.
212
212
213 """
213 """
214
214
215 try:
215 try:
216 fp = h5py.File(filename,'r')
216 fp = h5py.File(filename,'r')
217 grp1 = fp['Data']
217 grp1 = fp['Data']
218
218
219 except IOError:
219 except IOError:
220 traceback.print_exc()
220 traceback.print_exc()
221 raise IOError, "The file %s can't be opened" %(filename)
221 raise IOError, "The file %s can't be opened" %(filename)
222 #chino rata
222 #chino rata
223 #In case has utctime attribute
223 #In case has utctime attribute
224 grp2 = grp1['utctime']
224 grp2 = grp1['utctime']
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 thisUtcTime = grp2.value[0]
226 thisUtcTime = grp2.value[0]
227
227
228 fp.close()
228 fp.close()
229
229
230 if self.timezone == 'lt':
230 if self.timezone == 'lt':
231 thisUtcTime -= 5*3600
231 thisUtcTime -= 5*3600
232
232
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 thisDate = thisDatetime.date()
235 thisDate = thisDatetime.date()
236 thisTime = thisDatetime.time()
236 thisTime = thisDatetime.time()
237
237
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240
240
241 #General case
241 #General case
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 #-----------o----------------------------o-----------
243 #-----------o----------------------------o-----------
244 # startTime endTime
244 # startTime endTime
245
245
246 if endTime >= startTime:
246 if endTime >= startTime:
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 return thisDatetime
249 return thisDatetime
250 return None
250 return None
251
251
252 #If endTime < startTime then endTime belongs to the next day
252 #If endTime < startTime then endTime belongs to the next day
253 #<<<<<<<<<<<o o>>>>>>>>>>>
253 #<<<<<<<<<<<o o>>>>>>>>>>>
254 #-----------o----------------------------o-----------
254 #-----------o----------------------------o-----------
255 # endTime startTime
255 # endTime startTime
256
256
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 return None
258 return None
259
259
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 return None
261 return None
262
262
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 return None
264 return None
265
265
266 return thisDatetime
266 return thisDatetime
267
267
268 def __setNextFileOffline(self):
268 def __setNextFileOffline(self):
269
269
270 self.fileIndex += 1
270 self.fileIndex += 1
271 idFile = self.fileIndex
271 idFile = self.fileIndex
272
272
273 if not(idFile < len(self.filenameList)):
273 if not(idFile < len(self.filenameList)):
274 print "No more Files"
274 print "No more Files"
275 return 0
275 return 0
276
276
277 filename = self.filenameList[idFile]
277 filename = self.filenameList[idFile]
278
278
279 filePointer = h5py.File(filename,'r')
279 filePointer = h5py.File(filename,'r')
280
280
281 self.filename = filename
281 self.filename = filename
282
282
283 self.fp = filePointer
283 self.fp = filePointer
284
284
285 print "Setting the file: %s"%self.filename
285 print "Setting the file: %s"%self.filename
286
286
287 # self.__readMetadata()
287 # self.__readMetadata()
288 self.__setBlockList()
288 self.__setBlockList()
289 self.__readData()
289 self.__readData()
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 self.blockIndex = 0
292 self.blockIndex = 0
293 return 1
293 return 1
294
294
295 def __setBlockList(self):
295 def __setBlockList(self):
296 '''
296 '''
297 Selects the data within the times defined
297 Selects the data within the times defined
298
298
299 self.fp
299 self.fp
300 self.startTime
300 self.startTime
301 self.endTime
301 self.endTime
302
302
303 self.blockList
303 self.blockList
304 self.blocksPerFile
304 self.blocksPerFile
305
305
306 '''
306 '''
307 fp = self.fp
307 fp = self.fp
308 startTime = self.startTime
308 startTime = self.startTime
309 endTime = self.endTime
309 endTime = self.endTime
310
310
311 grp = fp['Data']
311 grp = fp['Data']
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313
313
314 #ERROOOOR
314 #ERROOOOR
315 if self.timezone == 'lt':
315 if self.timezone == 'lt':
316 thisUtcTime -= 5*3600
316 thisUtcTime -= 5*3600
317
317
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319
319
320 thisDate = thisDatetime.date()
320 thisDate = thisDatetime.date()
321 thisTime = thisDatetime.time()
321 thisTime = thisDatetime.time()
322
322
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325
325
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327
327
328 self.blockList = ind
328 self.blockList = ind
329 self.blocksPerFile = len(ind)
329 self.blocksPerFile = len(ind)
330
330
331 return
331 return
332
332
333 def __readMetadata(self):
333 def __readMetadata(self):
334 '''
334 '''
335 Reads Metadata
335 Reads Metadata
336
336
337 self.pathMeta
337 self.pathMeta
338
338
339 self.listShapes
339 self.listShapes
340 self.listMetaname
340 self.listMetaname
341 self.listMeta
341 self.listMeta
342
342
343 '''
343 '''
344
344
345 # grp = self.fp['Data']
345 # grp = self.fp['Data']
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 #
347 #
348 # if pathMeta == self.pathMeta:
348 # if pathMeta == self.pathMeta:
349 # return
349 # return
350 # else:
350 # else:
351 # self.pathMeta = pathMeta
351 # self.pathMeta = pathMeta
352 #
352 #
353 # filePointer = h5py.File(self.pathMeta,'r')
353 # filePointer = h5py.File(self.pathMeta,'r')
354 # groupPointer = filePointer['Metadata']
354 # groupPointer = filePointer['Metadata']
355
355
356 filename = self.filenameList[0]
356 filename = self.filenameList[0]
357
357
358 fp = h5py.File(filename,'r')
358 fp = h5py.File(filename,'r')
359
359
360 gp = fp['Metadata']
360 gp = fp['Metadata']
361
361
362 listMetaname = []
362 listMetaname = []
363 listMetadata = []
363 listMetadata = []
364 for item in gp.items():
364 for item in gp.items():
365 name = item[0]
365 name = item[0]
366
366
367 if name=='array dimensions':
367 if name=='array dimensions':
368 table = gp[name][:]
368 table = gp[name][:]
369 listShapes = {}
369 listShapes = {}
370 for shapes in table:
370 for shapes in table:
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 else:
372 else:
373 data = gp[name].value
373 data = gp[name].value
374 listMetaname.append(name)
374 listMetaname.append(name)
375 listMetadata.append(data)
375 listMetadata.append(data)
376
376
377 # if name=='type':
377 # if name=='type':
378 # self.__initDataOut(data)
378 # self.__initDataOut(data)
379
379
380 self.listShapes = listShapes
380 self.listShapes = listShapes
381 self.listMetaname = listMetaname
381 self.listMetaname = listMetaname
382 self.listMeta = listMetadata
382 self.listMeta = listMetadata
383
383
384 fp.close()
384 fp.close()
385 return
385 return
386
386
387 def __readData(self):
387 def __readData(self):
388 grp = self.fp['Data']
388 grp = self.fp['Data']
389 listdataname = []
389 listdataname = []
390 listdata = []
390 listdata = []
391
391
392 for item in grp.items():
392 for item in grp.items():
393 name = item[0]
393 name = item[0]
394 listdataname.append(name)
394 listdataname.append(name)
395
395
396 array = self.__setDataArray(grp[name],self.listShapes[name])
396 array = self.__setDataArray(grp[name],self.listShapes[name])
397 listdata.append(array)
397 listdata.append(array)
398
398
399 self.listDataname = listdataname
399 self.listDataname = listdataname
400 self.listData = listdata
400 self.listData = listdata
401 return
401 return
402
402
403 def __setDataArray(self, dataset, shapes):
403 def __setDataArray(self, dataset, shapes):
404
404
405 nDims = shapes[0]
405 nDims = shapes[0]
406
406
407 nDim2 = shapes[1] #Dimension 0
407 nDim2 = shapes[1] #Dimension 0
408
408
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410
410
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412
412
413 mode = shapes[4] #Mode of storing
413 mode = shapes[4] #Mode of storing
414
414
415 blockList = self.blockList
415 blockList = self.blockList
416
416
417 blocksPerFile = self.blocksPerFile
417 blocksPerFile = self.blocksPerFile
418
418
419 #Depending on what mode the data was stored
419 #Depending on what mode the data was stored
420 if mode == 0: #Divided in channels
420 if mode == 0: #Divided in channels
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 if mode == 1: #Divided in parameter
422 if mode == 1: #Divided in parameter
423 strds = 'table'
423 strds = 'table'
424 nDatas = nDim1
424 nDatas = nDim1
425 newShapes = (blocksPerFile,nDim2,nDim0)
425 newShapes = (blocksPerFile,nDim2,nDim0)
426 elif mode==2: #Concatenated in a table
426 elif mode==2: #Concatenated in a table
427 strds = 'table0'
427 strds = 'table0'
428 arrayData = dataset[strds].value
428 arrayData = dataset[strds].value
429 #Selecting part of the dataset
429 #Selecting part of the dataset
430 utctime = arrayData[:,0]
430 utctime = arrayData[:,0]
431 u, indices = numpy.unique(utctime, return_index=True)
431 u, indices = numpy.unique(utctime, return_index=True)
432
432
433 if blockList.size != indices.size:
433 if blockList.size != indices.size:
434 indMin = indices[blockList[0]]
434 indMin = indices[blockList[0]]
435 if blockList[1] + 1 >= indices.size:
435 if blockList[1] + 1 >= indices.size:
436 arrayData = arrayData[indMin:,:]
436 arrayData = arrayData[indMin:,:]
437 else:
437 else:
438 indMax = indices[blockList[1] + 1]
438 indMax = indices[blockList[1] + 1]
439 arrayData = arrayData[indMin:indMax,:]
439 arrayData = arrayData[indMin:indMax,:]
440 return arrayData
440 return arrayData
441
441
442 # One dimension
442 # One dimension
443 if nDims == 0:
443 if nDims == 0:
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445
445
446 # Two dimensions
446 # Two dimensions
447 elif nDims == 2:
447 elif nDims == 2:
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 newShapes = (blocksPerFile,nDim0)
449 newShapes = (blocksPerFile,nDim0)
450 nDatas = nDim1
450 nDatas = nDim1
451
451
452 for i in range(nDatas):
452 for i in range(nDatas):
453 data = dataset[strds + str(i)].value
453 data = dataset[strds + str(i)].value
454 arrayData[:,i,:] = data[blockList,:]
454 arrayData[:,i,:] = data[blockList,:]
455
455
456 # Three dimensions
456 # Three dimensions
457 else:
457 else:
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 for i in range(nDatas):
459 for i in range(nDatas):
460
460
461 data = dataset[strds + str(i)].value
461 data = dataset[strds + str(i)].value
462
462
463 for b in range(blockList.size):
463 for b in range(blockList.size):
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465
465
466 return arrayData
466 return arrayData
467
467
468 def __setDataOut(self):
468 def __setDataOut(self):
469 listMeta = self.listMeta
469 listMeta = self.listMeta
470 listMetaname = self.listMetaname
470 listMetaname = self.listMetaname
471 listDataname = self.listDataname
471 listDataname = self.listDataname
472 listData = self.listData
472 listData = self.listData
473 listShapes = self.listShapes
473 listShapes = self.listShapes
474
474
475 blockIndex = self.blockIndex
475 blockIndex = self.blockIndex
476 # blockList = self.blockList
476 # blockList = self.blockList
477
477
478 for i in range(len(listMeta)):
478 for i in range(len(listMeta)):
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
480
480
481 for j in range(len(listData)):
481 for j in range(len(listData)):
482 nShapes = listShapes[listDataname[j]][0]
482 nShapes = listShapes[listDataname[j]][0]
483 mode = listShapes[listDataname[j]][4]
483 mode = listShapes[listDataname[j]][4]
484 if nShapes == 1:
484 if nShapes == 1:
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 elif nShapes > 1:
486 elif nShapes > 1:
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 elif mode==0:
488 elif mode==0:
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 #Mode Meteors
490 #Mode Meteors
491 elif mode ==2:
491 elif mode ==2:
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 setattr(self.dataOut, listDataname[j], selectedData)
493 setattr(self.dataOut, listDataname[j], selectedData)
494 return
494 return
495
495
496 def __selectDataMode2(self, data, blockIndex):
496 def __selectDataMode2(self, data, blockIndex):
497 utctime = data[:,0]
497 utctime = data[:,0]
498 aux, indices = numpy.unique(utctime, return_inverse=True)
498 aux, indices = numpy.unique(utctime, return_inverse=True)
499 selInd = numpy.where(indices == blockIndex)[0]
499 selInd = numpy.where(indices == blockIndex)[0]
500 selData = data[selInd,:]
500 selData = data[selInd,:]
501
501
502 return selData
502 return selData
503
503
504 def getData(self):
504 def getData(self):
505
505
506 # if self.flagNoMoreFiles:
506 # if self.flagNoMoreFiles:
507 # self.dataOut.flagNoData = True
507 # self.dataOut.flagNoData = True
508 # print 'Process finished'
508 # print 'Process finished'
509 # return 0
509 # return 0
510 #
510 #
511 if self.blockIndex==self.blocksPerFile:
511 if self.blockIndex==self.blocksPerFile:
512 if not( self.__setNextFileOffline() ):
512 if not( self.__setNextFileOffline() ):
513 self.dataOut.flagNoData = True
513 self.dataOut.flagNoData = True
514 return 0
514 return 0
515
515
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 # self.dataOut.flagNoData = True
517 # self.dataOut.flagNoData = True
518 # return 0
518 # return 0
519 # self.__readData()
519 # self.__readData()
520 self.__setDataOut()
520 self.__setDataOut()
521 self.dataOut.flagNoData = False
521 self.dataOut.flagNoData = False
522
522
523 self.blockIndex += 1
523 self.blockIndex += 1
524
524
525 return
525 return
526
526
527 def run(self, **kwargs):
527 def run(self, **kwargs):
528
528
529 if not(self.isConfig):
529 if not(self.isConfig):
530 self.setup(**kwargs)
530 self.setup(**kwargs)
531 # self.setObjProperties()
531 # self.setObjProperties()
532 self.isConfig = True
532 self.isConfig = True
533
533
534 self.getData()
534 self.getData()
535
535
536 return
536 return
537
537
538 class ParamWriter(Operation):
538 class ParamWriter(Operation):
539 '''
539 '''
540 HDF5 Writer, stores parameters data in HDF5 format files
540 HDF5 Writer, stores parameters data in HDF5 format files
541
541
542 path: path where the files will be stored
542 path: path where the files will be stored
543
543
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545
545
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547
547
548 metadataList: list of attributes that will be stored as metadata
548 metadataList: list of attributes that will be stored as metadata
549
549
550 dataList: list of attributes that will be stores as data
550 dataList: list of attributes that will be stores as data
551
551
552 '''
552 '''
553
553
554
554
555 ext = ".hdf5"
555 ext = ".hdf5"
556
556
557 optchar = "D"
557 optchar = "D"
558
558
559 metaoptchar = "M"
559 metaoptchar = "M"
560
560
561 metaFile = None
561 metaFile = None
562
562
563 filename = None
563 filename = None
564
564
565 path = None
565 path = None
566
566
567 setFile = None
567 setFile = None
568
568
569 fp = None
569 fp = None
570
570
571 grp = None
571 grp = None
572
572
573 ds = None
573 ds = None
574
574
575 firsttime = True
575 firsttime = True
576
576
577 #Configurations
577 #Configurations
578
578
579 blocksPerFile = None
579 blocksPerFile = None
580
580
581 blockIndex = None
581 blockIndex = None
582
582
583 dataOut = None
583 dataOut = None
584
584
585 #Data Arrays
585 #Data Arrays
586
586
587 dataList = None
587 dataList = None
588
588
589 metadataList = None
589 metadataList = None
590
590
591 # arrayDim = None
591 # arrayDim = None
592
592
593 dsList = None #List of dictionaries with dataset properties
593 dsList = None #List of dictionaries with dataset properties
594
594
595 tableDim = None
595 tableDim = None
596
596
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598
598
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600
600
601 currentDay = None
601 currentDay = None
602
602
603 lastTime = None
603 lastTime = None
604
604
605 def __init__(self, **kwargs):
605 def __init__(self, **kwargs):
606 Operation.__init__(self, **kwargs)
606 Operation.__init__(self, **kwargs)
607 self.isConfig = False
607 self.isConfig = False
608 return
608 return
609
609
610 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
610 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 self.path = path
611 self.path = path
612 self.blocksPerFile = blocksPerFile
612 self.blocksPerFile = blocksPerFile
613 self.metadataList = metadataList
613 self.metadataList = metadataList
614 self.dataList = dataList
614 self.dataList = dataList
615 self.dataOut = dataOut
615 self.dataOut = dataOut
616 self.mode = mode
616 self.mode = mode
617
617
618 if self.mode is not None:
618 if self.mode is not None:
619 self.mode = numpy.zeros(len(self.dataList)) + mode
619 self.mode = numpy.zeros(len(self.dataList)) + mode
620 else:
620 else:
621 self.mode = numpy.ones(len(self.dataList))
621 self.mode = numpy.ones(len(self.dataList))
622
622
623 arrayDim = numpy.zeros((len(self.dataList),5))
623 arrayDim = numpy.zeros((len(self.dataList),5))
624
624
625 #Table dimensions
625 #Table dimensions
626 dtype0 = self.dtype
626 dtype0 = self.dtype
627 tableList = []
627 tableList = []
628
628
629 #Dictionary and list of tables
629 #Dictionary and list of tables
630 dsList = []
630 dsList = []
631
631
632 for i in range(len(self.dataList)):
632 for i in range(len(self.dataList)):
633 dsDict = {}
633 dsDict = {}
634 dataAux = getattr(self.dataOut, self.dataList[i])
634 dataAux = getattr(self.dataOut, self.dataList[i])
635 dsDict['variable'] = self.dataList[i]
635 dsDict['variable'] = self.dataList[i]
636 #--------------------- Conditionals ------------------------
636 #--------------------- Conditionals ------------------------
637 #There is no data
637 #There is no data
638 if dataAux is None:
638 if dataAux is None:
639 return 0
639 return 0
640
640
641 #Not array, just a number
641 #Not array, just a number
642 #Mode 0
642 #Mode 0
643 if type(dataAux)==float or type(dataAux)==int:
643 if type(dataAux)==float or type(dataAux)==int:
644 dsDict['mode'] = 0
644 dsDict['mode'] = 0
645 dsDict['nDim'] = 0
645 dsDict['nDim'] = 0
646 arrayDim[i,0] = 0
646 arrayDim[i,0] = 0
647 dsList.append(dsDict)
647 dsList.append(dsDict)
648
648
649 #Mode 2: meteors
649 #Mode 2: meteors
650 elif mode[i] == 2:
650 elif mode[i] == 2:
651 # dsDict['nDim'] = 0
651 # dsDict['nDim'] = 0
652 dsDict['dsName'] = 'table0'
652 dsDict['dsName'] = 'table0'
653 dsDict['mode'] = 2 # Mode meteors
653 dsDict['mode'] = 2 # Mode meteors
654 dsDict['shape'] = dataAux.shape[-1]
654 dsDict['shape'] = dataAux.shape[-1]
655 dsDict['nDim'] = 0
655 dsDict['nDim'] = 0
656 dsDict['dsNumber'] = 1
656 dsDict['dsNumber'] = 1
657
657
658 arrayDim[i,3] = dataAux.shape[-1]
658 arrayDim[i,3] = dataAux.shape[-1]
659 arrayDim[i,4] = mode[i] #Mode the data was stored
659 arrayDim[i,4] = mode[i] #Mode the data was stored
660
660
661 dsList.append(dsDict)
661 dsList.append(dsDict)
662
662
663 #Mode 1
663 #Mode 1
664 else:
664 else:
665 arrayDim0 = dataAux.shape #Data dimensions
665 arrayDim0 = dataAux.shape #Data dimensions
666 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
666 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
667 arrayDim[i,4] = mode[i] #Mode the data was stored
667 arrayDim[i,4] = mode[i] #Mode the data was stored
668
668
669 strtable = 'table'
669 strtable = 'table'
670 dsDict['mode'] = 1 # Mode parameters
670 dsDict['mode'] = 1 # Mode parameters
671
671
672 # Three-dimension arrays
672 # Three-dimension arrays
673 if len(arrayDim0) == 3:
673 if len(arrayDim0) == 3:
674 arrayDim[i,1:-1] = numpy.array(arrayDim0)
674 arrayDim[i,1:-1] = numpy.array(arrayDim0)
675 nTables = int(arrayDim[i,2])
675 nTables = int(arrayDim[i,2])
676 dsDict['dsNumber'] = nTables
676 dsDict['dsNumber'] = nTables
677 dsDict['shape'] = arrayDim[i,2:4]
677 dsDict['shape'] = arrayDim[i,2:4]
678 dsDict['nDim'] = 3
678 dsDict['nDim'] = 3
679
679
680 for j in range(nTables):
680 for j in range(nTables):
681 dsDict = dsDict.copy()
681 dsDict = dsDict.copy()
682 dsDict['dsName'] = strtable + str(j)
682 dsDict['dsName'] = strtable + str(j)
683 dsList.append(dsDict)
683 dsList.append(dsDict)
684
684
685 # Two-dimension arrays
685 # Two-dimension arrays
686 elif len(arrayDim0) == 2:
686 elif len(arrayDim0) == 2:
687 arrayDim[i,2:-1] = numpy.array(arrayDim0)
687 arrayDim[i,2:-1] = numpy.array(arrayDim0)
688 nTables = int(arrayDim[i,2])
688 nTables = int(arrayDim[i,2])
689 dsDict['dsNumber'] = nTables
689 dsDict['dsNumber'] = nTables
690 dsDict['shape'] = arrayDim[i,3]
690 dsDict['shape'] = arrayDim[i,3]
691 dsDict['nDim'] = 2
691 dsDict['nDim'] = 2
692
692
693 for j in range(nTables):
693 for j in range(nTables):
694 dsDict = dsDict.copy()
694 dsDict = dsDict.copy()
695 dsDict['dsName'] = strtable + str(j)
695 dsDict['dsName'] = strtable + str(j)
696 dsList.append(dsDict)
696 dsList.append(dsDict)
697
697
698 # One-dimension arrays
698 # One-dimension arrays
699 elif len(arrayDim0) == 1:
699 elif len(arrayDim0) == 1:
700 arrayDim[i,3] = arrayDim0[0]
700 arrayDim[i,3] = arrayDim0[0]
701 dsDict['shape'] = arrayDim0[0]
701 dsDict['shape'] = arrayDim0[0]
702 dsDict['dsNumber'] = 1
702 dsDict['dsNumber'] = 1
703 dsDict['dsName'] = strtable + str(0)
703 dsDict['dsName'] = strtable + str(0)
704 dsDict['nDim'] = 1
704 dsDict['nDim'] = 1
705 dsList.append(dsDict)
705 dsList.append(dsDict)
706
706
707 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
707 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
708 tableList.append(table)
708 tableList.append(table)
709
709
710 # self.arrayDim = arrayDim
710 # self.arrayDim = arrayDim
711 self.dsList = dsList
711 self.dsList = dsList
712 self.tableDim = numpy.array(tableList, dtype = dtype0)
712 self.tableDim = numpy.array(tableList, dtype = dtype0)
713 self.blockIndex = 0
713 self.blockIndex = 0
714
714
715 timeTuple = time.localtime(dataOut.utctime)
715 timeTuple = time.localtime(dataOut.utctime)
716 self.currentDay = timeTuple.tm_yday
716 self.currentDay = timeTuple.tm_yday
717 return 1
717 return 1
718
718
719 def putMetadata(self):
719 def putMetadata(self):
720
720
721 fp = self.createMetadataFile()
721 fp = self.createMetadataFile()
722 self.writeMetadata(fp)
722 self.writeMetadata(fp)
723 fp.close()
723 fp.close()
724 return
724 return
725
725
726 def createMetadataFile(self):
726 def createMetadataFile(self):
727 ext = self.ext
727 ext = self.ext
728 path = self.path
728 path = self.path
729 setFile = self.setFile
729 setFile = self.setFile
730
730
731 timeTuple = time.localtime(self.dataOut.utctime)
731 timeTuple = time.localtime(self.dataOut.utctime)
732
732
733 subfolder = ''
733 subfolder = ''
734 fullpath = os.path.join( path, subfolder )
734 fullpath = os.path.join( path, subfolder )
735
735
736 if not( os.path.exists(fullpath) ):
736 if not( os.path.exists(fullpath) ):
737 os.mkdir(fullpath)
737 os.mkdir(fullpath)
738 setFile = -1 #inicializo mi contador de seteo
738 setFile = -1 #inicializo mi contador de seteo
739
739
740 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
740 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
741 fullpath = os.path.join( path, subfolder )
741 fullpath = os.path.join( path, subfolder )
742
742
743 if not( os.path.exists(fullpath) ):
743 if not( os.path.exists(fullpath) ):
744 os.mkdir(fullpath)
744 os.mkdir(fullpath)
745 setFile = -1 #inicializo mi contador de seteo
745 setFile = -1 #inicializo mi contador de seteo
746
746
747 else:
747 else:
748 filesList = os.listdir( fullpath )
748 filesList = os.listdir( fullpath )
749 filesList = sorted( filesList, key=str.lower )
749 filesList = sorted( filesList, key=str.lower )
750 if len( filesList ) > 0:
750 if len( filesList ) > 0:
751 filesList = [k for k in filesList if 'M' in k]
751 filesList = [k for k in filesList if 'M' in k]
752 filen = filesList[-1]
752 filen = filesList[-1]
753 # el filename debera tener el siguiente formato
753 # el filename debera tener el siguiente formato
754 # 0 1234 567 89A BCDE (hex)
754 # 0 1234 567 89A BCDE (hex)
755 # x YYYY DDD SSS .ext
755 # x YYYY DDD SSS .ext
756 if isNumber( filen[8:11] ):
756 if isNumber( filen[8:11] ):
757 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
757 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
758 else:
758 else:
759 setFile = -1
759 setFile = -1
760 else:
760 else:
761 setFile = -1 #inicializo mi contador de seteo
761 setFile = -1 #inicializo mi contador de seteo
762
762
763 if self.setType is None:
763 if self.setType is None:
764 setFile += 1
764 setFile += 1
765 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
765 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
766 timeTuple.tm_year,
766 timeTuple.tm_year,
767 timeTuple.tm_yday,
767 timeTuple.tm_yday,
768 setFile,
768 setFile,
769 ext )
769 ext )
770 else:
770 else:
771 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
771 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
772 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
772 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
773 timeTuple.tm_year,
773 timeTuple.tm_year,
774 timeTuple.tm_yday,
774 timeTuple.tm_yday,
775 setFile,
775 setFile,
776 ext )
776 ext )
777
777
778 filename = os.path.join( path, subfolder, file )
778 filename = os.path.join( path, subfolder, file )
779 self.metaFile = file
779 self.metaFile = file
780 #Setting HDF5 File
780 #Setting HDF5 File
781 fp = h5py.File(filename,'w')
781 fp = h5py.File(filename,'w')
782
782
783 return fp
783 return fp
784
784
785 def writeMetadata(self, fp):
785 def writeMetadata(self, fp):
786
786
787 grp = fp.create_group("Metadata")
787 grp = fp.create_group("Metadata")
788 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
788 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
789
789
790 for i in range(len(self.metadataList)):
790 for i in range(len(self.metadataList)):
791 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
791 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
792 return
792 return
793
793
794 def timeFlag(self):
794 def timeFlag(self):
795 currentTime = self.dataOut.utctime
795 currentTime = self.dataOut.utctime
796
796
797 if self.lastTime is None:
797 if self.lastTime is None:
798 self.lastTime = currentTime
798 self.lastTime = currentTime
799
799
800 #Day
800 #Day
801 timeTuple = time.localtime(currentTime)
801 timeTuple = time.localtime(currentTime)
802 dataDay = timeTuple.tm_yday
802 dataDay = timeTuple.tm_yday
803
803
804 #Time
804 #Time
805 timeDiff = currentTime - self.lastTime
805 timeDiff = currentTime - self.lastTime
806
806
807 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
807 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
808 if dataDay != self.currentDay:
808 if dataDay != self.currentDay:
809 self.currentDay = dataDay
809 self.currentDay = dataDay
810 return True
810 return True
811 elif timeDiff > 3*60*60:
811 elif timeDiff > 3*60*60:
812 self.lastTime = currentTime
812 self.lastTime = currentTime
813 return True
813 return True
814 else:
814 else:
815 self.lastTime = currentTime
815 self.lastTime = currentTime
816 return False
816 return False
817
817
818 def setNextFile(self):
818 def setNextFile(self):
819
819
820 ext = self.ext
820 ext = self.ext
821 path = self.path
821 path = self.path
822 setFile = self.setFile
822 setFile = self.setFile
823 mode = self.mode
823 mode = self.mode
824
824
825 timeTuple = time.localtime(self.dataOut.utctime)
825 timeTuple = time.localtime(self.dataOut.utctime)
826 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
826 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
827
827
828 fullpath = os.path.join( path, subfolder )
828 fullpath = os.path.join( path, subfolder )
829
829
830 if os.path.exists(fullpath):
830 if os.path.exists(fullpath):
831 filesList = os.listdir( fullpath )
831 filesList = os.listdir( fullpath )
832 filesList = [k for k in filesList if 'D' in k]
832 filesList = [k for k in filesList if 'M' in k]
833 if len( filesList ) > 0:
833 if len( filesList ) > 0:
834 filesList = sorted( filesList, key=str.lower )
834 filesList = sorted( filesList, key=str.lower )
835 filen = filesList[-1]
835 filen = filesList[-1]
836 # el filename debera tener el siguiente formato
836 # el filename debera tener el siguiente formato
837 # 0 1234 567 89A BCDE (hex)
837 # 0 1234 567 89A BCDE (hex)
838 # x YYYY DDD SSS .ext
838 # x YYYY DDD SSS .ext
839 if isNumber( filen[8:11] ):
839 if isNumber( filen[8:11] ):
840 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
840 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
841 else:
841 else:
842 setFile = -1
842 setFile = -1
843 else:
843 else:
844 setFile = -1 #inicializo mi contador de seteo
844 setFile = -1 #inicializo mi contador de seteo
845 else:
845 else:
846 os.makedirs(fullpath)
846 os.makedirs(fullpath)
847 setFile = -1 #inicializo mi contador de seteo
847 setFile = -1 #inicializo mi contador de seteo
848
848
849 if self.setType is None:
849 if self.setType is None:
850 setFile += 1
850 setFile += 1
851 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
851 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
852 timeTuple.tm_year,
852 timeTuple.tm_year,
853 timeTuple.tm_yday,
853 timeTuple.tm_yday,
854 setFile,
854 setFile,
855 ext )
855 ext )
856 else:
856 else:
857 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
857 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
858 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
858 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
859 timeTuple.tm_year,
859 timeTuple.tm_year,
860 timeTuple.tm_yday,
860 timeTuple.tm_yday,
861 setFile,
861 setFile,
862 ext )
862 ext )
863
863
864 filename = os.path.join( path, subfolder, file )
864 filename = os.path.join( path, subfolder, file )
865
865
866 #Setting HDF5 File
866 #Setting HDF5 File
867 fp = h5py.File(filename,'w')
867 fp = h5py.File(filename,'w')
868 #write metadata
868 #write metadata
869 self.writeMetadata(fp)
869 self.writeMetadata(fp)
870 #Write data
870 #Write data
871 grp = fp.create_group("Data")
871 grp = fp.create_group("Data")
872 # grp.attrs['metadata'] = self.metaFile
872 # grp.attrs['metadata'] = self.metaFile
873
873
874 # grp.attrs['blocksPerFile'] = 0
874 # grp.attrs['blocksPerFile'] = 0
875 ds = []
875 ds = []
876 data = []
876 data = []
877 dsList = self.dsList
877 dsList = self.dsList
878 i = 0
878 i = 0
879 while i < len(dsList):
879 while i < len(dsList):
880 dsInfo = dsList[i]
880 dsInfo = dsList[i]
881 #One-dimension data
881 #One-dimension data
882 if dsInfo['mode'] == 0:
882 if dsInfo['mode'] == 0:
883 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
883 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
884 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
884 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
885 ds.append(ds0)
885 ds.append(ds0)
886 data.append([])
886 data.append([])
887 i += 1
887 i += 1
888 continue
888 continue
889 # nDimsForDs.append(nDims[i])
889 # nDimsForDs.append(nDims[i])
890
890
891 elif dsInfo['mode'] == 2:
891 elif dsInfo['mode'] == 2:
892 grp0 = grp.create_group(dsInfo['variable'])
892 grp0 = grp.create_group(dsInfo['variable'])
893 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
893 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
894 ds.append(ds0)
894 ds.append(ds0)
895 data.append([])
895 data.append([])
896 i += 1
896 i += 1
897 continue
897 continue
898
898
899 elif dsInfo['mode'] == 1:
899 elif dsInfo['mode'] == 1:
900 grp0 = grp.create_group(dsInfo['variable'])
900 grp0 = grp.create_group(dsInfo['variable'])
901
901
902 for j in range(dsInfo['dsNumber']):
902 for j in range(dsInfo['dsNumber']):
903 dsInfo = dsList[i]
903 dsInfo = dsList[i]
904 tableName = dsInfo['dsName']
904 tableName = dsInfo['dsName']
905 shape = int(dsInfo['shape'])
905 shape = int(dsInfo['shape'])
906
906
907 if dsInfo['nDim'] == 3:
907 if dsInfo['nDim'] == 3:
908 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
908 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
909 else:
909 else:
910 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
910 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
911
911
912 ds.append(ds0)
912 ds.append(ds0)
913 data.append([])
913 data.append([])
914 i += 1
914 i += 1
915 # nDimsForDs.append(nDims[i])
915 # nDimsForDs.append(nDims[i])
916
916
917 fp.flush()
917 fp.flush()
918 fp.close()
918 fp.close()
919
919
920 # self.nDatas = nDatas
920 # self.nDatas = nDatas
921 # self.nDims = nDims
921 # self.nDims = nDims
922 # self.nDimsForDs = nDimsForDs
922 # self.nDimsForDs = nDimsForDs
923 #Saving variables
923 #Saving variables
924 print 'Writing the file: %s'%filename
924 print 'Writing the file: %s'%filename
925 self.filename = filename
925 self.filename = filename
926 # self.fp = fp
926 # self.fp = fp
927 # self.grp = grp
927 # self.grp = grp
928 # self.grp.attrs.modify('nRecords', 1)
928 # self.grp.attrs.modify('nRecords', 1)
929 self.ds = ds
929 self.ds = ds
930 self.data = data
930 self.data = data
931 # self.setFile = setFile
931 # self.setFile = setFile
932 self.firsttime = True
932 self.firsttime = True
933 self.blockIndex = 0
933 self.blockIndex = 0
934 return
934 return
935
935
936 def putData(self):
936 def putData(self):
937
937
938 if self.blockIndex == self.blocksPerFile or self.timeFlag():
938 if self.blockIndex == self.blocksPerFile or self.timeFlag():
939 self.setNextFile()
939 self.setNextFile()
940
940
941 # if not self.firsttime:
941 # if not self.firsttime:
942 self.readBlock()
942 self.readBlock()
943 self.setBlock() #Prepare data to be written
943 self.setBlock() #Prepare data to be written
944 self.writeBlock() #Write data
944 self.writeBlock() #Write data
945
945
946 return
946 return
947
947
948 def readBlock(self):
948 def readBlock(self):
949
949
950 '''
950 '''
951 data Array configured
951 data Array configured
952
952
953
953
954 self.data
954 self.data
955 '''
955 '''
956 dsList = self.dsList
956 dsList = self.dsList
957 ds = self.ds
957 ds = self.ds
958 #Setting HDF5 File
958 #Setting HDF5 File
959 fp = h5py.File(self.filename,'r+')
959 fp = h5py.File(self.filename,'r+')
960 grp = fp["Data"]
960 grp = fp["Data"]
961 ind = 0
961 ind = 0
962
962
963 # grp.attrs['blocksPerFile'] = 0
963 # grp.attrs['blocksPerFile'] = 0
964 while ind < len(dsList):
964 while ind < len(dsList):
965 dsInfo = dsList[ind]
965 dsInfo = dsList[ind]
966
966
967 if dsInfo['mode'] == 0:
967 if dsInfo['mode'] == 0:
968 ds0 = grp[dsInfo['variable']]
968 ds0 = grp[dsInfo['variable']]
969 ds[ind] = ds0
969 ds[ind] = ds0
970 ind += 1
970 ind += 1
971 else:
971 else:
972
972
973 grp0 = grp[dsInfo['variable']]
973 grp0 = grp[dsInfo['variable']]
974
974
975 for j in range(dsInfo['dsNumber']):
975 for j in range(dsInfo['dsNumber']):
976 dsInfo = dsList[ind]
976 dsInfo = dsList[ind]
977 ds0 = grp0[dsInfo['dsName']]
977 ds0 = grp0[dsInfo['dsName']]
978 ds[ind] = ds0
978 ds[ind] = ds0
979 ind += 1
979 ind += 1
980
980
981 self.fp = fp
981 self.fp = fp
982 self.grp = grp
982 self.grp = grp
983 self.ds = ds
983 self.ds = ds
984
984
985 return
985 return
986
986
987 def setBlock(self):
987 def setBlock(self):
988 '''
988 '''
989 data Array configured
989 data Array configured
990
990
991
991
992 self.data
992 self.data
993 '''
993 '''
994 #Creating Arrays
994 #Creating Arrays
995 dsList = self.dsList
995 dsList = self.dsList
996 data = self.data
996 data = self.data
997 ind = 0
997 ind = 0
998
998
999 while ind < len(dsList):
999 while ind < len(dsList):
1000 dsInfo = dsList[ind]
1000 dsInfo = dsList[ind]
1001 dataAux = getattr(self.dataOut, dsInfo['variable'])
1001 dataAux = getattr(self.dataOut, dsInfo['variable'])
1002
1002
1003 mode = dsInfo['mode']
1003 mode = dsInfo['mode']
1004 nDim = dsInfo['nDim']
1004 nDim = dsInfo['nDim']
1005
1005
1006 if mode == 0 or mode == 2 or nDim == 1:
1006 if mode == 0 or mode == 2 or nDim == 1:
1007 data[ind] = dataAux
1007 data[ind] = dataAux
1008 ind += 1
1008 ind += 1
1009 # elif nDim == 1:
1009 # elif nDim == 1:
1010 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1010 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1011 # ind += 1
1011 # ind += 1
1012 elif nDim == 2:
1012 elif nDim == 2:
1013 for j in range(dsInfo['dsNumber']):
1013 for j in range(dsInfo['dsNumber']):
1014 data[ind] = dataAux[j,:]
1014 data[ind] = dataAux[j,:]
1015 ind += 1
1015 ind += 1
1016 elif nDim == 3:
1016 elif nDim == 3:
1017 for j in range(dsInfo['dsNumber']):
1017 for j in range(dsInfo['dsNumber']):
1018 data[ind] = dataAux[:,j,:]
1018 data[ind] = dataAux[:,j,:]
1019 ind += 1
1019 ind += 1
1020
1020
1021 self.data = data
1021 self.data = data
1022 return
1022 return
1023
1023
1024 def writeBlock(self):
1024 def writeBlock(self):
1025 '''
1025 '''
1026 Saves the block in the HDF5 file
1026 Saves the block in the HDF5 file
1027 '''
1027 '''
1028 dsList = self.dsList
1028 dsList = self.dsList
1029
1029
1030 for i in range(len(self.ds)):
1030 for i in range(len(self.ds)):
1031 dsInfo = dsList[i]
1031 dsInfo = dsList[i]
1032 nDim = dsInfo['nDim']
1032 nDim = dsInfo['nDim']
1033 mode = dsInfo['mode']
1033 mode = dsInfo['mode']
1034
1034
1035 # First time
1035 # First time
1036 if self.firsttime:
1036 if self.firsttime:
1037 # self.ds[i].resize(self.data[i].shape)
1037 # self.ds[i].resize(self.data[i].shape)
1038 # self.ds[i][self.blockIndex,:] = self.data[i]
1038 # self.ds[i][self.blockIndex,:] = self.data[i]
1039 if type(self.data[i]) == numpy.ndarray:
1039 if type(self.data[i]) == numpy.ndarray:
1040
1040
1041 if nDim == 3:
1041 if nDim == 3:
1042 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1042 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1043 self.ds[i].resize(self.data[i].shape)
1043 self.ds[i].resize(self.data[i].shape)
1044 if mode == 2:
1044 if mode == 2:
1045 self.ds[i].resize(self.data[i].shape)
1045 self.ds[i].resize(self.data[i].shape)
1046 self.ds[i][:] = self.data[i]
1046 self.ds[i][:] = self.data[i]
1047 else:
1047 else:
1048
1048
1049 # From second time
1049 # From second time
1050 # Meteors!
1050 # Meteors!
1051 if mode == 2:
1051 if mode == 2:
1052 dataShape = self.data[i].shape
1052 dataShape = self.data[i].shape
1053 dsShape = self.ds[i].shape
1053 dsShape = self.ds[i].shape
1054 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1054 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1055 self.ds[i][dsShape[0]:,:] = self.data[i]
1055 self.ds[i][dsShape[0]:,:] = self.data[i]
1056 # No dimension
1056 # No dimension
1057 elif mode == 0:
1057 elif mode == 0:
1058 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1058 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1059 self.ds[i][0,-1] = self.data[i]
1059 self.ds[i][0,-1] = self.data[i]
1060 # One dimension
1060 # One dimension
1061 elif nDim == 1:
1061 elif nDim == 1:
1062 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1062 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1063 self.ds[i][-1,:] = self.data[i]
1063 self.ds[i][-1,:] = self.data[i]
1064 # Two dimension
1064 # Two dimension
1065 elif nDim == 2:
1065 elif nDim == 2:
1066 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1066 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1067 self.ds[i][self.blockIndex,:] = self.data[i]
1067 self.ds[i][self.blockIndex,:] = self.data[i]
1068 # Three dimensions
1068 # Three dimensions
1069 elif nDim == 3:
1069 elif nDim == 3:
1070 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1070 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1071 self.ds[i][:,:,-1] = self.data[i]
1071 self.ds[i][:,:,-1] = self.data[i]
1072
1072
1073 self.firsttime = False
1073 self.firsttime = False
1074 self.blockIndex += 1
1074 self.blockIndex += 1
1075
1075
1076 #Close to save changes
1076 #Close to save changes
1077 self.fp.flush()
1077 self.fp.flush()
1078 self.fp.close()
1078 self.fp.close()
1079 return
1079 return
1080
1080
1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1082
1082
1083 if not(self.isConfig):
1083 if not(self.isConfig):
1084 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1084 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1085 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1085 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1086
1086
1087 if not(flagdata):
1087 if not(flagdata):
1088 return
1088 return
1089
1089
1090 self.isConfig = True
1090 self.isConfig = True
1091 # self.putMetadata()
1091 # self.putMetadata()
1092 self.setNextFile()
1092 self.setNextFile()
1093
1093
1094 self.putData()
1094 self.putData()
1095 return
1095 return
General Comments 0
You need to be logged in to leave comments. Login now