##// END OF EJS Templates
schain
RSS
Set default value for setType in ParamWriter
Juan C. Espinoza -
r1222:4fb9a88ae50c
parent child
Show More
Side by Side Unified
Context file:
r1222:4fb9a88ae50c Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1095 +1,1097
1 import numpy
1 import numpy
2 import time
2 import time
3 import os
3 import os
4 import h5py
4 import h5py
5 import re
5 import re
6 import datetime
6 import datetime
7
7
8 from schainpy.model.data.jrodata import *
8 from schainpy.model.data.jrodata import *
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 # from jroIO_base import *
10 # from jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
11 from schainpy.model.io.jroIO_base import *
12 import schainpy
12 import schainpy
13
13
14
14
15 class ParamReader(ProcessingUnit):
15 class ParamReader(ProcessingUnit):
16 '''
16 '''
17 Reads HDF5 format files
17 Reads HDF5 format files
18
18
19 path
19 path
20
20
21 startDate
21 startDate
22
22
23 endDate
23 endDate
24
24
25 startTime
25 startTime
26
26
27 endTime
27 endTime
28 '''
28 '''
29
29
30 ext = ".hdf5"
30 ext = ".hdf5"
31
31
32 optchar = "D"
32 optchar = "D"
33
33
34 timezone = None
34 timezone = None
35
35
36 startTime = None
36 startTime = None
37
37
38 endTime = None
38 endTime = None
39
39
40 fileIndex = None
40 fileIndex = None
41
41
42 utcList = None #To select data in the utctime list
42 utcList = None #To select data in the utctime list
43
43
44 blockList = None #List to blocks to be read from the file
44 blockList = None #List to blocks to be read from the file
45
45
46 blocksPerFile = None #Number of blocks to be read
46 blocksPerFile = None #Number of blocks to be read
47
47
48 blockIndex = None
48 blockIndex = None
49
49
50 path = None
50 path = None
51
51
52 #List of Files
52 #List of Files
53
53
54 filenameList = None
54 filenameList = None
55
55
56 datetimeList = None
56 datetimeList = None
57
57
58 #Hdf5 File
58 #Hdf5 File
59
59
60 listMetaname = None
60 listMetaname = None
61
61
62 listMeta = None
62 listMeta = None
63
63
64 listDataname = None
64 listDataname = None
65
65
66 listData = None
66 listData = None
67
67
68 listShapes = None
68 listShapes = None
69
69
70 fp = None
70 fp = None
71
71
72 #dataOut reconstruction
72 #dataOut reconstruction
73
73
74 dataOut = None
74 dataOut = None
75
75
76
76
77 def __init__(self, **kwargs):
77 def __init__(self, **kwargs):
78 ProcessingUnit.__init__(self, **kwargs)
78 ProcessingUnit.__init__(self, **kwargs)
79 self.dataOut = Parameters()
79 self.dataOut = Parameters()
80 return
80 return
81
81
82 def setup(self, **kwargs):
82 def setup(self, **kwargs):
83
83
84 path = kwargs['path']
84 path = kwargs['path']
85 startDate = kwargs['startDate']
85 startDate = kwargs['startDate']
86 endDate = kwargs['endDate']
86 endDate = kwargs['endDate']
87 startTime = kwargs['startTime']
87 startTime = kwargs['startTime']
88 endTime = kwargs['endTime']
88 endTime = kwargs['endTime']
89 walk = kwargs['walk']
89 walk = kwargs['walk']
90 if kwargs.has_key('ext'):
90 if kwargs.has_key('ext'):
91 ext = kwargs['ext']
91 ext = kwargs['ext']
92 else:
92 else:
93 ext = '.hdf5'
93 ext = '.hdf5'
94 if kwargs.has_key('timezone'):
94 if kwargs.has_key('timezone'):
95 self.timezone = kwargs['timezone']
95 self.timezone = kwargs['timezone']
96 else:
96 else:
97 self.timezone = 'lt'
97 self.timezone = 'lt'
98
98
99 print "[Reading] Searching files in offline mode ..."
99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 startTime=startTime, endTime=endTime,
101 startTime=startTime, endTime=endTime,
102 ext=ext, walk=walk)
102 ext=ext, walk=walk)
103
103
104 if not(filenameList):
104 if not(filenameList):
105 print "There is no files into the folder: %s"%(path)
105 print "There is no files into the folder: %s"%(path)
106 sys.exit(-1)
106 sys.exit(-1)
107
107
108 self.fileIndex = -1
108 self.fileIndex = -1
109 self.startTime = startTime
109 self.startTime = startTime
110 self.endTime = endTime
110 self.endTime = endTime
111
111
112 self.__readMetadata()
112 self.__readMetadata()
113
113
114 self.__setNextFileOffline()
114 self.__setNextFileOffline()
115
115
116 return
116 return
117
117
118 def searchFilesOffLine(self,
118 def searchFilesOffLine(self,
119 path,
119 path,
120 startDate=None,
120 startDate=None,
121 endDate=None,
121 endDate=None,
122 startTime=datetime.time(0,0,0),
122 startTime=datetime.time(0,0,0),
123 endTime=datetime.time(23,59,59),
123 endTime=datetime.time(23,59,59),
124 ext='.hdf5',
124 ext='.hdf5',
125 walk=True):
125 walk=True):
126
126
127 expLabel = ''
127 expLabel = ''
128 self.filenameList = []
128 self.filenameList = []
129 self.datetimeList = []
129 self.datetimeList = []
130
130
131 pathList = []
131 pathList = []
132
132
133 JRODataObj = JRODataReader()
133 JRODataObj = JRODataReader()
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135
135
136 if dateList == []:
136 if dateList == []:
137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 datetime.datetime.combine(startDate,startTime).ctime(),
138 datetime.datetime.combine(startDate,startTime).ctime(),
139 datetime.datetime.combine(endDate,endTime).ctime())
139 datetime.datetime.combine(endDate,endTime).ctime())
140
140
141 return None, None
141 return None, None
142
142
143 if len(dateList) > 1:
143 if len(dateList) > 1:
144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
145 else:
145 else:
146 print "[Reading] data was found for the date %s" %(dateList[0])
146 print "[Reading] data was found for the date %s" %(dateList[0])
147
147
148 filenameList = []
148 filenameList = []
149 datetimeList = []
149 datetimeList = []
150
150
151 #----------------------------------------------------------------------------------
151 #----------------------------------------------------------------------------------
152
152
153 for thisPath in pathList:
153 for thisPath in pathList:
154 # thisPath = pathList[pathDict[file]]
154 # thisPath = pathList[pathDict[file]]
155
155
156 fileList = glob.glob1(thisPath, "*%s" %ext)
156 fileList = glob.glob1(thisPath, "*%s" %ext)
157 fileList.sort()
157 fileList.sort()
158
158
159 for file in fileList:
159 for file in fileList:
160
160
161 filename = os.path.join(thisPath,file)
161 filename = os.path.join(thisPath,file)
162
162
163 if not isFileInDateRange(filename, startDate, endDate):
163 if not isFileInDateRange(filename, startDate, endDate):
164 continue
164 continue
165
165
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167
167
168 if not(thisDatetime):
168 if not(thisDatetime):
169 continue
169 continue
170
170
171 filenameList.append(filename)
171 filenameList.append(filename)
172 datetimeList.append(thisDatetime)
172 datetimeList.append(thisDatetime)
173
173
174 if not(filenameList):
174 if not(filenameList):
175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
176 return None, None
176 return None, None
177
177
178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
179 print
179 print
180
180
181 # for i in range(len(filenameList)):
181 # for i in range(len(filenameList)):
182 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
182 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183
183
184 self.filenameList = filenameList
184 self.filenameList = filenameList
185 self.datetimeList = datetimeList
185 self.datetimeList = datetimeList
186
186
187 return pathList, filenameList
187 return pathList, filenameList
188
188
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190
190
191 """
191 """
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193
193
194 Inputs:
194 Inputs:
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196
196
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
197 startDate : fecha inicial del rango seleccionado en formato datetime.date
198
198
199 endDate : fecha final del rango seleccionado en formato datetime.date
199 endDate : fecha final del rango seleccionado en formato datetime.date
200
200
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202
202
203 endTime : tiempo final del rango seleccionado en formato datetime.time
203 endTime : tiempo final del rango seleccionado en formato datetime.time
204
204
205 Return:
205 Return:
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 fecha especificado, de lo contrario retorna False.
207 fecha especificado, de lo contrario retorna False.
208
208
209 Excepciones:
209 Excepciones:
210 Si el archivo no existe o no puede ser abierto
210 Si el archivo no existe o no puede ser abierto
211 Si la cabecera no puede ser leida.
211 Si la cabecera no puede ser leida.
212
212
213 """
213 """
214
214
215 try:
215 try:
216 fp = h5py.File(filename,'r')
216 fp = h5py.File(filename,'r')
217 grp1 = fp['Data']
217 grp1 = fp['Data']
218
218
219 except IOError:
219 except IOError:
220 traceback.print_exc()
220 traceback.print_exc()
221 raise IOError, "The file %s can't be opened" %(filename)
221 raise IOError, "The file %s can't be opened" %(filename)
222 #chino rata
222 #chino rata
223 #In case has utctime attribute
223 #In case has utctime attribute
224 grp2 = grp1['utctime']
224 grp2 = grp1['utctime']
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 thisUtcTime = grp2.value[0]
226 thisUtcTime = grp2.value[0]
227
227
228 fp.close()
228 fp.close()
229
229
230 if self.timezone == 'lt':
230 if self.timezone == 'lt':
231 thisUtcTime -= 5*3600
231 thisUtcTime -= 5*3600
232
232
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 thisDate = thisDatetime.date()
235 thisDate = thisDatetime.date()
236 thisTime = thisDatetime.time()
236 thisTime = thisDatetime.time()
237
237
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240
240
241 #General case
241 #General case
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 #-----------o----------------------------o-----------
243 #-----------o----------------------------o-----------
244 # startTime endTime
244 # startTime endTime
245
245
246 if endTime >= startTime:
246 if endTime >= startTime:
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 return thisDatetime
249 return thisDatetime
250 return None
250 return None
251
251
252 #If endTime < startTime then endTime belongs to the next day
252 #If endTime < startTime then endTime belongs to the next day
253 #<<<<<<<<<<<o o>>>>>>>>>>>
253 #<<<<<<<<<<<o o>>>>>>>>>>>
254 #-----------o----------------------------o-----------
254 #-----------o----------------------------o-----------
255 # endTime startTime
255 # endTime startTime
256
256
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 return None
258 return None
259
259
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 return None
261 return None
262
262
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 return None
264 return None
265
265
266 return thisDatetime
266 return thisDatetime
267
267
268 def __setNextFileOffline(self):
268 def __setNextFileOffline(self):
269
269
270 self.fileIndex += 1
270 self.fileIndex += 1
271 idFile = self.fileIndex
271 idFile = self.fileIndex
272
272
273 if not(idFile < len(self.filenameList)):
273 if not(idFile < len(self.filenameList)):
274 print "No more Files"
274 print "No more Files"
275 return 0
275 return 0
276
276
277 filename = self.filenameList[idFile]
277 filename = self.filenameList[idFile]
278
278
279 filePointer = h5py.File(filename,'r')
279 filePointer = h5py.File(filename,'r')
280
280
281 self.filename = filename
281 self.filename = filename
282
282
283 self.fp = filePointer
283 self.fp = filePointer
284
284
285 print "Setting the file: %s"%self.filename
285 print "Setting the file: %s"%self.filename
286
286
287 # self.__readMetadata()
287 # self.__readMetadata()
288 self.__setBlockList()
288 self.__setBlockList()
289 self.__readData()
289 self.__readData()
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
291 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 self.blockIndex = 0
292 self.blockIndex = 0
293 return 1
293 return 1
294
294
295 def __setBlockList(self):
295 def __setBlockList(self):
296 '''
296 '''
297 Selects the data within the times defined
297 Selects the data within the times defined
298
298
299 self.fp
299 self.fp
300 self.startTime
300 self.startTime
301 self.endTime
301 self.endTime
302
302
303 self.blockList
303 self.blockList
304 self.blocksPerFile
304 self.blocksPerFile
305
305
306 '''
306 '''
307 fp = self.fp
307 fp = self.fp
308 startTime = self.startTime
308 startTime = self.startTime
309 endTime = self.endTime
309 endTime = self.endTime
310
310
311 grp = fp['Data']
311 grp = fp['Data']
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313
313
314 #ERROOOOR
314 #ERROOOOR
315 if self.timezone == 'lt':
315 if self.timezone == 'lt':
316 thisUtcTime -= 5*3600
316 thisUtcTime -= 5*3600
317
317
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319
319
320 thisDate = thisDatetime.date()
320 thisDate = thisDatetime.date()
321 thisTime = thisDatetime.time()
321 thisTime = thisDatetime.time()
322
322
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325
325
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327
327
328 self.blockList = ind
328 self.blockList = ind
329 self.blocksPerFile = len(ind)
329 self.blocksPerFile = len(ind)
330
330
331 return
331 return
332
332
333 def __readMetadata(self):
333 def __readMetadata(self):
334 '''
334 '''
335 Reads Metadata
335 Reads Metadata
336
336
337 self.pathMeta
337 self.pathMeta
338
338
339 self.listShapes
339 self.listShapes
340 self.listMetaname
340 self.listMetaname
341 self.listMeta
341 self.listMeta
342
342
343 '''
343 '''
344
344
345 # grp = self.fp['Data']
345 # grp = self.fp['Data']
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 #
347 #
348 # if pathMeta == self.pathMeta:
348 # if pathMeta == self.pathMeta:
349 # return
349 # return
350 # else:
350 # else:
351 # self.pathMeta = pathMeta
351 # self.pathMeta = pathMeta
352 #
352 #
353 # filePointer = h5py.File(self.pathMeta,'r')
353 # filePointer = h5py.File(self.pathMeta,'r')
354 # groupPointer = filePointer['Metadata']
354 # groupPointer = filePointer['Metadata']
355
355
356 filename = self.filenameList[0]
356 filename = self.filenameList[0]
357
357
358 fp = h5py.File(filename,'r')
358 fp = h5py.File(filename,'r')
359
359
360 gp = fp['Metadata']
360 gp = fp['Metadata']
361
361
362 listMetaname = []
362 listMetaname = []
363 listMetadata = []
363 listMetadata = []
364 for item in gp.items():
364 for item in gp.items():
365 name = item[0]
365 name = item[0]
366
366
367 if name=='array dimensions':
367 if name=='array dimensions':
368 table = gp[name][:]
368 table = gp[name][:]
369 listShapes = {}
369 listShapes = {}
370 for shapes in table:
370 for shapes in table:
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 else:
372 else:
373 data = gp[name].value
373 data = gp[name].value
374 listMetaname.append(name)
374 listMetaname.append(name)
375 listMetadata.append(data)
375 listMetadata.append(data)
376
376
377 # if name=='type':
377 # if name=='type':
378 # self.__initDataOut(data)
378 # self.__initDataOut(data)
379
379
380 self.listShapes = listShapes
380 self.listShapes = listShapes
381 self.listMetaname = listMetaname
381 self.listMetaname = listMetaname
382 self.listMeta = listMetadata
382 self.listMeta = listMetadata
383
383
384 fp.close()
384 fp.close()
385 return
385 return
386
386
387 def __readData(self):
387 def __readData(self):
388 grp = self.fp['Data']
388 grp = self.fp['Data']
389 listdataname = []
389 listdataname = []
390 listdata = []
390 listdata = []
391
391
392 for item in grp.items():
392 for item in grp.items():
393 name = item[0]
393 name = item[0]
394 listdataname.append(name)
394 listdataname.append(name)
395
395
396 array = self.__setDataArray(grp[name],self.listShapes[name])
396 array = self.__setDataArray(grp[name],self.listShapes[name])
397 listdata.append(array)
397 listdata.append(array)
398
398
399 self.listDataname = listdataname
399 self.listDataname = listdataname
400 self.listData = listdata
400 self.listData = listdata
401 return
401 return
402
402
403 def __setDataArray(self, dataset, shapes):
403 def __setDataArray(self, dataset, shapes):
404
404
405 nDims = shapes[0]
405 nDims = shapes[0]
406
406
407 nDim2 = shapes[1] #Dimension 0
407 nDim2 = shapes[1] #Dimension 0
408
408
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410
410
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412
412
413 mode = shapes[4] #Mode of storing
413 mode = shapes[4] #Mode of storing
414
414
415 blockList = self.blockList
415 blockList = self.blockList
416
416
417 blocksPerFile = self.blocksPerFile
417 blocksPerFile = self.blocksPerFile
418
418
419 #Depending on what mode the data was stored
419 #Depending on what mode the data was stored
420 if mode == 0: #Divided in channels
420 if mode == 0: #Divided in channels
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 if mode == 1: #Divided in parameter
422 if mode == 1: #Divided in parameter
423 strds = 'table'
423 strds = 'table'
424 nDatas = nDim1
424 nDatas = nDim1
425 newShapes = (blocksPerFile,nDim2,nDim0)
425 newShapes = (blocksPerFile,nDim2,nDim0)
426 elif mode==2: #Concatenated in a table
426 elif mode==2: #Concatenated in a table
427 strds = 'table0'
427 strds = 'table0'
428 arrayData = dataset[strds].value
428 arrayData = dataset[strds].value
429 #Selecting part of the dataset
429 #Selecting part of the dataset
430 utctime = arrayData[:,0]
430 utctime = arrayData[:,0]
431 u, indices = numpy.unique(utctime, return_index=True)
431 u, indices = numpy.unique(utctime, return_index=True)
432
432
433 if blockList.size != indices.size:
433 if blockList.size != indices.size:
434 indMin = indices[blockList[0]]
434 indMin = indices[blockList[0]]
435 if blockList[1] + 1 >= indices.size:
435 if blockList[1] + 1 >= indices.size:
436 arrayData = arrayData[indMin:,:]
436 arrayData = arrayData[indMin:,:]
437 else:
437 else:
438 indMax = indices[blockList[1] + 1]
438 indMax = indices[blockList[1] + 1]
439 arrayData = arrayData[indMin:indMax,:]
439 arrayData = arrayData[indMin:indMax,:]
440 return arrayData
440 return arrayData
441
441
442 # One dimension
442 # One dimension
443 if nDims == 0:
443 if nDims == 0:
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445
445
446 # Two dimensions
446 # Two dimensions
447 elif nDims == 2:
447 elif nDims == 2:
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 newShapes = (blocksPerFile,nDim0)
449 newShapes = (blocksPerFile,nDim0)
450 nDatas = nDim1
450 nDatas = nDim1
451
451
452 for i in range(nDatas):
452 for i in range(nDatas):
453 data = dataset[strds + str(i)].value
453 data = dataset[strds + str(i)].value
454 arrayData[:,i,:] = data[blockList,:]
454 arrayData[:,i,:] = data[blockList,:]
455
455
456 # Three dimensions
456 # Three dimensions
457 else:
457 else:
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 for i in range(nDatas):
459 for i in range(nDatas):
460
460
461 data = dataset[strds + str(i)].value
461 data = dataset[strds + str(i)].value
462
462
463 for b in range(blockList.size):
463 for b in range(blockList.size):
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465
465
466 return arrayData
466 return arrayData
467
467
468 def __setDataOut(self):
468 def __setDataOut(self):
469 listMeta = self.listMeta
469 listMeta = self.listMeta
470 listMetaname = self.listMetaname
470 listMetaname = self.listMetaname
471 listDataname = self.listDataname
471 listDataname = self.listDataname
472 listData = self.listData
472 listData = self.listData
473 listShapes = self.listShapes
473 listShapes = self.listShapes
474
474
475 blockIndex = self.blockIndex
475 blockIndex = self.blockIndex
476 # blockList = self.blockList
476 # blockList = self.blockList
477
477
478 for i in range(len(listMeta)):
478 for i in range(len(listMeta)):
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
479 setattr(self.dataOut,listMetaname[i],listMeta[i])
480
480
481 for j in range(len(listData)):
481 for j in range(len(listData)):
482 nShapes = listShapes[listDataname[j]][0]
482 nShapes = listShapes[listDataname[j]][0]
483 mode = listShapes[listDataname[j]][4]
483 mode = listShapes[listDataname[j]][4]
484 if nShapes == 1:
484 if nShapes == 1:
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 elif nShapes > 1:
486 elif nShapes > 1:
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 elif mode==0:
488 elif mode==0:
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 #Mode Meteors
490 #Mode Meteors
491 elif mode ==2:
491 elif mode ==2:
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 setattr(self.dataOut, listDataname[j], selectedData)
493 setattr(self.dataOut, listDataname[j], selectedData)
494 return
494 return
495
495
496 def __selectDataMode2(self, data, blockIndex):
496 def __selectDataMode2(self, data, blockIndex):
497 utctime = data[:,0]
497 utctime = data[:,0]
498 aux, indices = numpy.unique(utctime, return_inverse=True)
498 aux, indices = numpy.unique(utctime, return_inverse=True)
499 selInd = numpy.where(indices == blockIndex)[0]
499 selInd = numpy.where(indices == blockIndex)[0]
500 selData = data[selInd,:]
500 selData = data[selInd,:]
501
501
502 return selData
502 return selData
503
503
504 def getData(self):
504 def getData(self):
505
505
506 # if self.flagNoMoreFiles:
506 # if self.flagNoMoreFiles:
507 # self.dataOut.flagNoData = True
507 # self.dataOut.flagNoData = True
508 # print 'Process finished'
508 # print 'Process finished'
509 # return 0
509 # return 0
510 #
510 #
511 if self.blockIndex==self.blocksPerFile:
511 if self.blockIndex==self.blocksPerFile:
512 if not( self.__setNextFileOffline() ):
512 if not( self.__setNextFileOffline() ):
513 self.dataOut.flagNoData = True
513 self.dataOut.flagNoData = True
514 return 0
514 return 0
515
515
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 # self.dataOut.flagNoData = True
517 # self.dataOut.flagNoData = True
518 # return 0
518 # return 0
519 # self.__readData()
519 # self.__readData()
520 self.__setDataOut()
520 self.__setDataOut()
521 self.dataOut.flagNoData = False
521 self.dataOut.flagNoData = False
522
522
523 self.blockIndex += 1
523 self.blockIndex += 1
524
524
525 return
525 return
526
526
527 def run(self, **kwargs):
527 def run(self, **kwargs):
528
528
529 if not(self.isConfig):
529 if not(self.isConfig):
530 self.setup(**kwargs)
530 self.setup(**kwargs)
531 # self.setObjProperties()
531 # self.setObjProperties()
532 self.isConfig = True
532 self.isConfig = True
533
533
534 self.getData()
534 self.getData()
535
535
536 return
536 return
537
537
538 class ParamWriter(Operation):
538 class ParamWriter(Operation):
539 '''
539 '''
540 HDF5 Writer, stores parameters data in HDF5 format files
540 HDF5 Writer, stores parameters data in HDF5 format files
541
541
542 path: path where the files will be stored
542 path: path where the files will be stored
543
543
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545
545
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547
547
548 metadataList: list of attributes that will be stored as metadata
548 metadataList: list of attributes that will be stored as metadata
549
549
550 dataList: list of attributes that will be stores as data
550 dataList: list of attributes that will be stores as data
551
551
552 '''
552 '''
553
553
554
554
555 ext = ".hdf5"
555 ext = ".hdf5"
556
556
557 optchar = "D"
557 optchar = "D"
558
558
559 metaoptchar = "M"
559 metaoptchar = "M"
560
560
561 metaFile = None
561 metaFile = None
562
562
563 filename = None
563 filename = None
564
564
565 path = None
565 path = None
566
566
567 setFile = None
567 setFile = None
568
568
569 fp = None
569 fp = None
570
570
571 grp = None
571 grp = None
572
572
573 ds = None
573 ds = None
574
574
575 firsttime = True
575 firsttime = True
576
576
577 #Configurations
577 #Configurations
578
578
579 blocksPerFile = None
579 blocksPerFile = None
580
580
581 blockIndex = None
581 blockIndex = None
582
582
583 dataOut = None
583 dataOut = None
584
584
585 #Data Arrays
585 #Data Arrays
586
586
587 dataList = None
587 dataList = None
588
588
589 metadataList = None
589 metadataList = None
590
590
591 # arrayDim = None
591 # arrayDim = None
592
592
593 dsList = None #List of dictionaries with dataset properties
593 dsList = None #List of dictionaries with dataset properties
594
594
595 tableDim = None
595 tableDim = None
596
596
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598
598
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600
600
601 currentDay = None
601 currentDay = None
602
602
603 lastTime = None
603 lastTime = None
604
604
605 setType = None
606
605 def __init__(self, **kwargs):
607 def __init__(self, **kwargs):
606 Operation.__init__(self, **kwargs)
608 Operation.__init__(self, **kwargs)
607 self.isConfig = False
609 self.isConfig = False
608 return
610 return
609
611
610 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
612 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 self.path = path
613 self.path = path
612 self.blocksPerFile = blocksPerFile
614 self.blocksPerFile = blocksPerFile
613 self.metadataList = metadataList
615 self.metadataList = metadataList
614 self.dataList = dataList
616 self.dataList = dataList
615 self.dataOut = dataOut
617 self.dataOut = dataOut
616 self.mode = mode
618 self.mode = mode
617
619
618 if self.mode is not None:
620 if self.mode is not None:
619 self.mode = numpy.zeros(len(self.dataList)) + mode
621 self.mode = numpy.zeros(len(self.dataList)) + mode
620 else:
622 else:
621 self.mode = numpy.ones(len(self.dataList))
623 self.mode = numpy.ones(len(self.dataList))
622
624
623 arrayDim = numpy.zeros((len(self.dataList),5))
625 arrayDim = numpy.zeros((len(self.dataList),5))
624
626
625 #Table dimensions
627 #Table dimensions
626 dtype0 = self.dtype
628 dtype0 = self.dtype
627 tableList = []
629 tableList = []
628
630
629 #Dictionary and list of tables
631 #Dictionary and list of tables
630 dsList = []
632 dsList = []
631
633
632 for i in range(len(self.dataList)):
634 for i in range(len(self.dataList)):
633 dsDict = {}
635 dsDict = {}
634 dataAux = getattr(self.dataOut, self.dataList[i])
636 dataAux = getattr(self.dataOut, self.dataList[i])
635 dsDict['variable'] = self.dataList[i]
637 dsDict['variable'] = self.dataList[i]
636 #--------------------- Conditionals ------------------------
638 #--------------------- Conditionals ------------------------
637 #There is no data
639 #There is no data
638 if dataAux is None:
640 if dataAux is None:
639 return 0
641 return 0
640
642
641 #Not array, just a number
643 #Not array, just a number
642 #Mode 0
644 #Mode 0
643 if type(dataAux)==float or type(dataAux)==int:
645 if type(dataAux)==float or type(dataAux)==int:
644 dsDict['mode'] = 0
646 dsDict['mode'] = 0
645 dsDict['nDim'] = 0
647 dsDict['nDim'] = 0
646 arrayDim[i,0] = 0
648 arrayDim[i,0] = 0
647 dsList.append(dsDict)
649 dsList.append(dsDict)
648
650
649 #Mode 2: meteors
651 #Mode 2: meteors
650 elif mode[i] == 2:
652 elif mode[i] == 2:
651 # dsDict['nDim'] = 0
653 # dsDict['nDim'] = 0
652 dsDict['dsName'] = 'table0'
654 dsDict['dsName'] = 'table0'
653 dsDict['mode'] = 2 # Mode meteors
655 dsDict['mode'] = 2 # Mode meteors
654 dsDict['shape'] = dataAux.shape[-1]
656 dsDict['shape'] = dataAux.shape[-1]
655 dsDict['nDim'] = 0
657 dsDict['nDim'] = 0
656 dsDict['dsNumber'] = 1
658 dsDict['dsNumber'] = 1
657
659
658 arrayDim[i,3] = dataAux.shape[-1]
660 arrayDim[i,3] = dataAux.shape[-1]
659 arrayDim[i,4] = mode[i] #Mode the data was stored
661 arrayDim[i,4] = mode[i] #Mode the data was stored
660
662
661 dsList.append(dsDict)
663 dsList.append(dsDict)
662
664
663 #Mode 1
665 #Mode 1
664 else:
666 else:
665 arrayDim0 = dataAux.shape #Data dimensions
667 arrayDim0 = dataAux.shape #Data dimensions
666 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
668 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
667 arrayDim[i,4] = mode[i] #Mode the data was stored
669 arrayDim[i,4] = mode[i] #Mode the data was stored
668
670
669 strtable = 'table'
671 strtable = 'table'
670 dsDict['mode'] = 1 # Mode parameters
672 dsDict['mode'] = 1 # Mode parameters
671
673
672 # Three-dimension arrays
674 # Three-dimension arrays
673 if len(arrayDim0) == 3:
675 if len(arrayDim0) == 3:
674 arrayDim[i,1:-1] = numpy.array(arrayDim0)
676 arrayDim[i,1:-1] = numpy.array(arrayDim0)
675 nTables = int(arrayDim[i,2])
677 nTables = int(arrayDim[i,2])
676 dsDict['dsNumber'] = nTables
678 dsDict['dsNumber'] = nTables
677 dsDict['shape'] = arrayDim[i,2:4]
679 dsDict['shape'] = arrayDim[i,2:4]
678 dsDict['nDim'] = 3
680 dsDict['nDim'] = 3
679
681
680 for j in range(nTables):
682 for j in range(nTables):
681 dsDict = dsDict.copy()
683 dsDict = dsDict.copy()
682 dsDict['dsName'] = strtable + str(j)
684 dsDict['dsName'] = strtable + str(j)
683 dsList.append(dsDict)
685 dsList.append(dsDict)
684
686
685 # Two-dimension arrays
687 # Two-dimension arrays
686 elif len(arrayDim0) == 2:
688 elif len(arrayDim0) == 2:
687 arrayDim[i,2:-1] = numpy.array(arrayDim0)
689 arrayDim[i,2:-1] = numpy.array(arrayDim0)
688 nTables = int(arrayDim[i,2])
690 nTables = int(arrayDim[i,2])
689 dsDict['dsNumber'] = nTables
691 dsDict['dsNumber'] = nTables
690 dsDict['shape'] = arrayDim[i,3]
692 dsDict['shape'] = arrayDim[i,3]
691 dsDict['nDim'] = 2
693 dsDict['nDim'] = 2
692
694
693 for j in range(nTables):
695 for j in range(nTables):
694 dsDict = dsDict.copy()
696 dsDict = dsDict.copy()
695 dsDict['dsName'] = strtable + str(j)
697 dsDict['dsName'] = strtable + str(j)
696 dsList.append(dsDict)
698 dsList.append(dsDict)
697
699
698 # One-dimension arrays
700 # One-dimension arrays
699 elif len(arrayDim0) == 1:
701 elif len(arrayDim0) == 1:
700 arrayDim[i,3] = arrayDim0[0]
702 arrayDim[i,3] = arrayDim0[0]
701 dsDict['shape'] = arrayDim0[0]
703 dsDict['shape'] = arrayDim0[0]
702 dsDict['dsNumber'] = 1
704 dsDict['dsNumber'] = 1
703 dsDict['dsName'] = strtable + str(0)
705 dsDict['dsName'] = strtable + str(0)
704 dsDict['nDim'] = 1
706 dsDict['nDim'] = 1
705 dsList.append(dsDict)
707 dsList.append(dsDict)
706
708
707 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
709 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
708 tableList.append(table)
710 tableList.append(table)
709
711
710 # self.arrayDim = arrayDim
712 # self.arrayDim = arrayDim
711 self.dsList = dsList
713 self.dsList = dsList
712 self.tableDim = numpy.array(tableList, dtype = dtype0)
714 self.tableDim = numpy.array(tableList, dtype = dtype0)
713 self.blockIndex = 0
715 self.blockIndex = 0
714
716
715 timeTuple = time.localtime(dataOut.utctime)
717 timeTuple = time.localtime(dataOut.utctime)
716 self.currentDay = timeTuple.tm_yday
718 self.currentDay = timeTuple.tm_yday
717 return 1
719 return 1
718
720
719 def putMetadata(self):
721 def putMetadata(self):
720
722
721 fp = self.createMetadataFile()
723 fp = self.createMetadataFile()
722 self.writeMetadata(fp)
724 self.writeMetadata(fp)
723 fp.close()
725 fp.close()
724 return
726 return
725
727
726 def createMetadataFile(self):
728 def createMetadataFile(self):
727 ext = self.ext
729 ext = self.ext
728 path = self.path
730 path = self.path
729 setFile = self.setFile
731 setFile = self.setFile
730
732
731 timeTuple = time.localtime(self.dataOut.utctime)
733 timeTuple = time.localtime(self.dataOut.utctime)
732
734
733 subfolder = ''
735 subfolder = ''
734 fullpath = os.path.join( path, subfolder )
736 fullpath = os.path.join( path, subfolder )
735
737
736 if not( os.path.exists(fullpath) ):
738 if not( os.path.exists(fullpath) ):
737 os.mkdir(fullpath)
739 os.mkdir(fullpath)
738 setFile = -1 #inicializo mi contador de seteo
740 setFile = -1 #inicializo mi contador de seteo
739
741
740 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
742 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
741 fullpath = os.path.join( path, subfolder )
743 fullpath = os.path.join( path, subfolder )
742
744
743 if not( os.path.exists(fullpath) ):
745 if not( os.path.exists(fullpath) ):
744 os.mkdir(fullpath)
746 os.mkdir(fullpath)
745 setFile = -1 #inicializo mi contador de seteo
747 setFile = -1 #inicializo mi contador de seteo
746
748
747 else:
749 else:
748 filesList = os.listdir( fullpath )
750 filesList = os.listdir( fullpath )
749 filesList = sorted( filesList, key=str.lower )
751 filesList = sorted( filesList, key=str.lower )
750 if len( filesList ) > 0:
752 if len( filesList ) > 0:
751 filesList = [k for k in filesList if 'M' in k]
753 filesList = [k for k in filesList if 'M' in k]
752 filen = filesList[-1]
754 filen = filesList[-1]
753 # el filename debera tener el siguiente formato
755 # el filename debera tener el siguiente formato
754 # 0 1234 567 89A BCDE (hex)
756 # 0 1234 567 89A BCDE (hex)
755 # x YYYY DDD SSS .ext
757 # x YYYY DDD SSS .ext
756 if isNumber( filen[8:11] ):
758 if isNumber( filen[8:11] ):
757 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
759 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
758 else:
760 else:
759 setFile = -1
761 setFile = -1
760 else:
762 else:
761 setFile = -1 #inicializo mi contador de seteo
763 setFile = -1 #inicializo mi contador de seteo
762
764
763 if self.setType is None:
765 if self.setType is None:
764 setFile += 1
766 setFile += 1
765 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
767 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
766 timeTuple.tm_year,
768 timeTuple.tm_year,
767 timeTuple.tm_yday,
769 timeTuple.tm_yday,
768 setFile,
770 setFile,
769 ext )
771 ext )
770 else:
772 else:
771 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
773 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
772 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
774 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
773 timeTuple.tm_year,
775 timeTuple.tm_year,
774 timeTuple.tm_yday,
776 timeTuple.tm_yday,
775 setFile,
777 setFile,
776 ext )
778 ext )
777
779
778 filename = os.path.join( path, subfolder, file )
780 filename = os.path.join( path, subfolder, file )
779 self.metaFile = file
781 self.metaFile = file
780 #Setting HDF5 File
782 #Setting HDF5 File
781 fp = h5py.File(filename,'w')
783 fp = h5py.File(filename,'w')
782
784
783 return fp
785 return fp
784
786
785 def writeMetadata(self, fp):
787 def writeMetadata(self, fp):
786
788
787 grp = fp.create_group("Metadata")
789 grp = fp.create_group("Metadata")
788 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
790 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
789
791
790 for i in range(len(self.metadataList)):
792 for i in range(len(self.metadataList)):
791 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
793 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
792 return
794 return
793
795
794 def timeFlag(self):
796 def timeFlag(self):
795 currentTime = self.dataOut.utctime
797 currentTime = self.dataOut.utctime
796
798
797 if self.lastTime is None:
799 if self.lastTime is None:
798 self.lastTime = currentTime
800 self.lastTime = currentTime
799
801
800 #Day
802 #Day
801 timeTuple = time.localtime(currentTime)
803 timeTuple = time.localtime(currentTime)
802 dataDay = timeTuple.tm_yday
804 dataDay = timeTuple.tm_yday
803
805
804 #Time
806 #Time
805 timeDiff = currentTime - self.lastTime
807 timeDiff = currentTime - self.lastTime
806
808
807 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
809 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
808 if dataDay != self.currentDay:
810 if dataDay != self.currentDay:
809 self.currentDay = dataDay
811 self.currentDay = dataDay
810 return True
812 return True
811 elif timeDiff > 3*60*60:
813 elif timeDiff > 3*60*60:
812 self.lastTime = currentTime
814 self.lastTime = currentTime
813 return True
815 return True
814 else:
816 else:
815 self.lastTime = currentTime
817 self.lastTime = currentTime
816 return False
818 return False
817
819
818 def setNextFile(self):
820 def setNextFile(self):
819
821
820 ext = self.ext
822 ext = self.ext
821 path = self.path
823 path = self.path
822 setFile = self.setFile
824 setFile = self.setFile
823 mode = self.mode
825 mode = self.mode
824
826
825 timeTuple = time.localtime(self.dataOut.utctime)
827 timeTuple = time.localtime(self.dataOut.utctime)
826 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
828 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
827
829
828 fullpath = os.path.join( path, subfolder )
830 fullpath = os.path.join( path, subfolder )
829
831
830 if os.path.exists(fullpath):
832 if os.path.exists(fullpath):
831 filesList = os.listdir( fullpath )
833 filesList = os.listdir( fullpath )
832 filesList = [k for k in filesList if 'M' in k]
834 filesList = [k for k in filesList if 'M' in k]
833 if len( filesList ) > 0:
835 if len( filesList ) > 0:
834 filesList = sorted( filesList, key=str.lower )
836 filesList = sorted( filesList, key=str.lower )
835 filen = filesList[-1]
837 filen = filesList[-1]
836 # el filename debera tener el siguiente formato
838 # el filename debera tener el siguiente formato
837 # 0 1234 567 89A BCDE (hex)
839 # 0 1234 567 89A BCDE (hex)
838 # x YYYY DDD SSS .ext
840 # x YYYY DDD SSS .ext
839 if isNumber( filen[8:11] ):
841 if isNumber( filen[8:11] ):
840 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
842 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
841 else:
843 else:
842 setFile = -1
844 setFile = -1
843 else:
845 else:
844 setFile = -1 #inicializo mi contador de seteo
846 setFile = -1 #inicializo mi contador de seteo
845 else:
847 else:
846 os.makedirs(fullpath)
848 os.makedirs(fullpath)
847 setFile = -1 #inicializo mi contador de seteo
849 setFile = -1 #inicializo mi contador de seteo
848
850
849 if self.setType is None:
851 if self.setType is None:
850 setFile += 1
852 setFile += 1
851 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
853 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
852 timeTuple.tm_year,
854 timeTuple.tm_year,
853 timeTuple.tm_yday,
855 timeTuple.tm_yday,
854 setFile,
856 setFile,
855 ext )
857 ext )
856 else:
858 else:
857 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
859 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
858 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
860 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
859 timeTuple.tm_year,
861 timeTuple.tm_year,
860 timeTuple.tm_yday,
862 timeTuple.tm_yday,
861 setFile,
863 setFile,
862 ext )
864 ext )
863
865
864 filename = os.path.join( path, subfolder, file )
866 filename = os.path.join( path, subfolder, file )
865
867
866 #Setting HDF5 File
868 #Setting HDF5 File
867 fp = h5py.File(filename,'w')
869 fp = h5py.File(filename,'w')
868 #write metadata
870 #write metadata
869 self.writeMetadata(fp)
871 self.writeMetadata(fp)
870 #Write data
872 #Write data
871 grp = fp.create_group("Data")
873 grp = fp.create_group("Data")
872 # grp.attrs['metadata'] = self.metaFile
874 # grp.attrs['metadata'] = self.metaFile
873
875
874 # grp.attrs['blocksPerFile'] = 0
876 # grp.attrs['blocksPerFile'] = 0
875 ds = []
877 ds = []
876 data = []
878 data = []
877 dsList = self.dsList
879 dsList = self.dsList
878 i = 0
880 i = 0
879 while i < len(dsList):
881 while i < len(dsList):
880 dsInfo = dsList[i]
882 dsInfo = dsList[i]
881 #One-dimension data
883 #One-dimension data
882 if dsInfo['mode'] == 0:
884 if dsInfo['mode'] == 0:
883 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
885 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
884 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
886 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
885 ds.append(ds0)
887 ds.append(ds0)
886 data.append([])
888 data.append([])
887 i += 1
889 i += 1
888 continue
890 continue
889 # nDimsForDs.append(nDims[i])
891 # nDimsForDs.append(nDims[i])
890
892
891 elif dsInfo['mode'] == 2:
893 elif dsInfo['mode'] == 2:
892 grp0 = grp.create_group(dsInfo['variable'])
894 grp0 = grp.create_group(dsInfo['variable'])
893 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
895 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
894 ds.append(ds0)
896 ds.append(ds0)
895 data.append([])
897 data.append([])
896 i += 1
898 i += 1
897 continue
899 continue
898
900
899 elif dsInfo['mode'] == 1:
901 elif dsInfo['mode'] == 1:
900 grp0 = grp.create_group(dsInfo['variable'])
902 grp0 = grp.create_group(dsInfo['variable'])
901
903
902 for j in range(dsInfo['dsNumber']):
904 for j in range(dsInfo['dsNumber']):
903 dsInfo = dsList[i]
905 dsInfo = dsList[i]
904 tableName = dsInfo['dsName']
906 tableName = dsInfo['dsName']
905 shape = int(dsInfo['shape'])
907 shape = int(dsInfo['shape'])
906
908
907 if dsInfo['nDim'] == 3:
909 if dsInfo['nDim'] == 3:
908 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
910 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
909 else:
911 else:
910 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
912 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
911
913
912 ds.append(ds0)
914 ds.append(ds0)
913 data.append([])
915 data.append([])
914 i += 1
916 i += 1
915 # nDimsForDs.append(nDims[i])
917 # nDimsForDs.append(nDims[i])
916
918
917 fp.flush()
919 fp.flush()
918 fp.close()
920 fp.close()
919
921
920 # self.nDatas = nDatas
922 # self.nDatas = nDatas
921 # self.nDims = nDims
923 # self.nDims = nDims
922 # self.nDimsForDs = nDimsForDs
924 # self.nDimsForDs = nDimsForDs
923 #Saving variables
925 #Saving variables
924 print 'Writing the file: %s'%filename
926 print 'Writing the file: %s'%filename
925 self.filename = filename
927 self.filename = filename
926 # self.fp = fp
928 # self.fp = fp
927 # self.grp = grp
929 # self.grp = grp
928 # self.grp.attrs.modify('nRecords', 1)
930 # self.grp.attrs.modify('nRecords', 1)
929 self.ds = ds
931 self.ds = ds
930 self.data = data
932 self.data = data
931 # self.setFile = setFile
933 # self.setFile = setFile
932 self.firsttime = True
934 self.firsttime = True
933 self.blockIndex = 0
935 self.blockIndex = 0
934 return
936 return
935
937
936 def putData(self):
938 def putData(self):
937
939
938 if self.blockIndex == self.blocksPerFile or self.timeFlag():
940 if self.blockIndex == self.blocksPerFile or self.timeFlag():
939 self.setNextFile()
941 self.setNextFile()
940
942
941 # if not self.firsttime:
943 # if not self.firsttime:
942 self.readBlock()
944 self.readBlock()
943 self.setBlock() #Prepare data to be written
945 self.setBlock() #Prepare data to be written
944 self.writeBlock() #Write data
946 self.writeBlock() #Write data
945
947
946 return
948 return
947
949
948 def readBlock(self):
950 def readBlock(self):
949
951
950 '''
952 '''
951 data Array configured
953 data Array configured
952
954
953
955
954 self.data
956 self.data
955 '''
957 '''
956 dsList = self.dsList
958 dsList = self.dsList
957 ds = self.ds
959 ds = self.ds
958 #Setting HDF5 File
960 #Setting HDF5 File
959 fp = h5py.File(self.filename,'r+')
961 fp = h5py.File(self.filename,'r+')
960 grp = fp["Data"]
962 grp = fp["Data"]
961 ind = 0
963 ind = 0
962
964
963 # grp.attrs['blocksPerFile'] = 0
965 # grp.attrs['blocksPerFile'] = 0
964 while ind < len(dsList):
966 while ind < len(dsList):
965 dsInfo = dsList[ind]
967 dsInfo = dsList[ind]
966
968
967 if dsInfo['mode'] == 0:
969 if dsInfo['mode'] == 0:
968 ds0 = grp[dsInfo['variable']]
970 ds0 = grp[dsInfo['variable']]
969 ds[ind] = ds0
971 ds[ind] = ds0
970 ind += 1
972 ind += 1
971 else:
973 else:
972
974
973 grp0 = grp[dsInfo['variable']]
975 grp0 = grp[dsInfo['variable']]
974
976
975 for j in range(dsInfo['dsNumber']):
977 for j in range(dsInfo['dsNumber']):
976 dsInfo = dsList[ind]
978 dsInfo = dsList[ind]
977 ds0 = grp0[dsInfo['dsName']]
979 ds0 = grp0[dsInfo['dsName']]
978 ds[ind] = ds0
980 ds[ind] = ds0
979 ind += 1
981 ind += 1
980
982
981 self.fp = fp
983 self.fp = fp
982 self.grp = grp
984 self.grp = grp
983 self.ds = ds
985 self.ds = ds
984
986
985 return
987 return
986
988
987 def setBlock(self):
989 def setBlock(self):
988 '''
990 '''
989 data Array configured
991 data Array configured
990
992
991
993
992 self.data
994 self.data
993 '''
995 '''
994 #Creating Arrays
996 #Creating Arrays
995 dsList = self.dsList
997 dsList = self.dsList
996 data = self.data
998 data = self.data
997 ind = 0
999 ind = 0
998
1000
999 while ind < len(dsList):
1001 while ind < len(dsList):
1000 dsInfo = dsList[ind]
1002 dsInfo = dsList[ind]
1001 dataAux = getattr(self.dataOut, dsInfo['variable'])
1003 dataAux = getattr(self.dataOut, dsInfo['variable'])
1002
1004
1003 mode = dsInfo['mode']
1005 mode = dsInfo['mode']
1004 nDim = dsInfo['nDim']
1006 nDim = dsInfo['nDim']
1005
1007
1006 if mode == 0 or mode == 2 or nDim == 1:
1008 if mode == 0 or mode == 2 or nDim == 1:
1007 data[ind] = dataAux
1009 data[ind] = dataAux
1008 ind += 1
1010 ind += 1
1009 # elif nDim == 1:
1011 # elif nDim == 1:
1010 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1012 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1011 # ind += 1
1013 # ind += 1
1012 elif nDim == 2:
1014 elif nDim == 2:
1013 for j in range(dsInfo['dsNumber']):
1015 for j in range(dsInfo['dsNumber']):
1014 data[ind] = dataAux[j,:]
1016 data[ind] = dataAux[j,:]
1015 ind += 1
1017 ind += 1
1016 elif nDim == 3:
1018 elif nDim == 3:
1017 for j in range(dsInfo['dsNumber']):
1019 for j in range(dsInfo['dsNumber']):
1018 data[ind] = dataAux[:,j,:]
1020 data[ind] = dataAux[:,j,:]
1019 ind += 1
1021 ind += 1
1020
1022
1021 self.data = data
1023 self.data = data
1022 return
1024 return
1023
1025
1024 def writeBlock(self):
1026 def writeBlock(self):
1025 '''
1027 '''
1026 Saves the block in the HDF5 file
1028 Saves the block in the HDF5 file
1027 '''
1029 '''
1028 dsList = self.dsList
1030 dsList = self.dsList
1029
1031
1030 for i in range(len(self.ds)):
1032 for i in range(len(self.ds)):
1031 dsInfo = dsList[i]
1033 dsInfo = dsList[i]
1032 nDim = dsInfo['nDim']
1034 nDim = dsInfo['nDim']
1033 mode = dsInfo['mode']
1035 mode = dsInfo['mode']
1034
1036
1035 # First time
1037 # First time
1036 if self.firsttime:
1038 if self.firsttime:
1037 # self.ds[i].resize(self.data[i].shape)
1039 # self.ds[i].resize(self.data[i].shape)
1038 # self.ds[i][self.blockIndex,:] = self.data[i]
1040 # self.ds[i][self.blockIndex,:] = self.data[i]
1039 if type(self.data[i]) == numpy.ndarray:
1041 if type(self.data[i]) == numpy.ndarray:
1040
1042
1041 if nDim == 3:
1043 if nDim == 3:
1042 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1044 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1043 self.ds[i].resize(self.data[i].shape)
1045 self.ds[i].resize(self.data[i].shape)
1044 if mode == 2:
1046 if mode == 2:
1045 self.ds[i].resize(self.data[i].shape)
1047 self.ds[i].resize(self.data[i].shape)
1046 self.ds[i][:] = self.data[i]
1048 self.ds[i][:] = self.data[i]
1047 else:
1049 else:
1048
1050
1049 # From second time
1051 # From second time
1050 # Meteors!
1052 # Meteors!
1051 if mode == 2:
1053 if mode == 2:
1052 dataShape = self.data[i].shape
1054 dataShape = self.data[i].shape
1053 dsShape = self.ds[i].shape
1055 dsShape = self.ds[i].shape
1054 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1056 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1055 self.ds[i][dsShape[0]:,:] = self.data[i]
1057 self.ds[i][dsShape[0]:,:] = self.data[i]
1056 # No dimension
1058 # No dimension
1057 elif mode == 0:
1059 elif mode == 0:
1058 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1060 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1059 self.ds[i][0,-1] = self.data[i]
1061 self.ds[i][0,-1] = self.data[i]
1060 # One dimension
1062 # One dimension
1061 elif nDim == 1:
1063 elif nDim == 1:
1062 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1064 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1063 self.ds[i][-1,:] = self.data[i]
1065 self.ds[i][-1,:] = self.data[i]
1064 # Two dimension
1066 # Two dimension
1065 elif nDim == 2:
1067 elif nDim == 2:
1066 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1068 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1067 self.ds[i][self.blockIndex,:] = self.data[i]
1069 self.ds[i][self.blockIndex,:] = self.data[i]
1068 # Three dimensions
1070 # Three dimensions
1069 elif nDim == 3:
1071 elif nDim == 3:
1070 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1072 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1071 self.ds[i][:,:,-1] = self.data[i]
1073 self.ds[i][:,:,-1] = self.data[i]
1072
1074
1073 self.firsttime = False
1075 self.firsttime = False
1074 self.blockIndex += 1
1076 self.blockIndex += 1
1075
1077
1076 #Close to save changes
1078 #Close to save changes
1077 self.fp.flush()
1079 self.fp.flush()
1078 self.fp.close()
1080 self.fp.close()
1079 return
1081 return
1080
1082
1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1083 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1082
1084
1083 if not(self.isConfig):
1085 if not(self.isConfig):
1084 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1086 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1085 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1087 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1086
1088
1087 if not(flagdata):
1089 if not(flagdata):
1088 return
1090 return
1089
1091
1090 self.isConfig = True
1092 self.isConfig = True
1091 # self.putMetadata()
1093 # self.putMetadata()
1092 self.setNextFile()
1094 self.setNextFile()
1093
1095
1094 self.putData()
1096 self.putData()
1095 return
1097 return
General Comments 0
You need to be logged in to leave comments. Login now