##// END OF EJS Templates
schain-jc
RSS
Bug in ParamWriter not saving numpy types (int, float)
Juan C. Espinoza -
r1195:9f1881ac55a3
parent child
Show More
Side by Side Unified
Context file:
r1195:9f1881ac55a3 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1036 +1,1035
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import datetime
7 7
8 8 from schainpy.model.data.jrodata import *
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 10 # from .jroIO_base import *
11 11 from schainpy.model.io.jroIO_base import *
12 12 import schainpy
13 13 from schainpy.utils import log
14 14
15 15 @MPDecorator
16 16 class ParamReader(JRODataReader,ProcessingUnit):
17 17 '''
18 18 Reads HDF5 format files
19 19
20 20 path
21 21
22 22 startDate
23 23
24 24 endDate
25 25
26 26 startTime
27 27
28 28 endTime
29 29 '''
30 30
31 31 ext = ".hdf5"
32 32
33 33 optchar = "D"
34 34
35 35 timezone = None
36 36
37 37 startTime = None
38 38
39 39 endTime = None
40 40
41 41 fileIndex = None
42 42
43 43 utcList = None #To select data in the utctime list
44 44
45 45 blockList = None #List to blocks to be read from the file
46 46
47 47 blocksPerFile = None #Number of blocks to be read
48 48
49 49 blockIndex = None
50 50
51 51 path = None
52 52
53 53 #List of Files
54 54
55 55 filenameList = None
56 56
57 57 datetimeList = None
58 58
59 59 #Hdf5 File
60 60
61 61 listMetaname = None
62 62
63 63 listMeta = None
64 64
65 65 listDataname = None
66 66
67 67 listData = None
68 68
69 69 listShapes = None
70 70
71 71 fp = None
72 72
73 73 #dataOut reconstruction
74 74
75 75 dataOut = None
76 76
77 77
78 78 def __init__(self):#, **kwargs):
79 79 ProcessingUnit.__init__(self) #, **kwargs)
80 80 self.dataOut = Parameters()
81 81 return
82 82
83 83 def setup(self, **kwargs):
84 84
85 85 path = kwargs['path']
86 86 startDate = kwargs['startDate']
87 87 endDate = kwargs['endDate']
88 88 startTime = kwargs['startTime']
89 89 endTime = kwargs['endTime']
90 90 walk = kwargs['walk']
91 91 if 'ext' in kwargs:
92 92 ext = kwargs['ext']
93 93 else:
94 94 ext = '.hdf5'
95 95 if 'timezone' in kwargs:
96 96 self.timezone = kwargs['timezone']
97 97 else:
98 98 self.timezone = 'lt'
99 99
100 100 print("[Reading] Searching files in offline mode ...")
101 101 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
102 102 startTime=startTime, endTime=endTime,
103 103 ext=ext, walk=walk)
104 104
105 105 if not(filenameList):
106 106 print("There is no files into the folder: %s"%(path))
107 107 sys.exit(-1)
108 108
109 109 self.fileIndex = -1
110 110 self.startTime = startTime
111 111 self.endTime = endTime
112 112
113 113 self.__readMetadata()
114 114
115 115 self.__setNextFileOffline()
116 116
117 117 return
118 118
119 119 def searchFilesOffLine(self,
120 120 path,
121 121 startDate=None,
122 122 endDate=None,
123 123 startTime=datetime.time(0,0,0),
124 124 endTime=datetime.time(23,59,59),
125 125 ext='.hdf5',
126 126 walk=True):
127 127
128 128 expLabel = ''
129 129 self.filenameList = []
130 130 self.datetimeList = []
131 131
132 132 pathList = []
133 133
134 134 JRODataObj = JRODataReader()
135 135 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
136 136
137 137 if dateList == []:
138 138 print("[Reading] No *%s files in %s from %s to %s)"%(ext, path,
139 139 datetime.datetime.combine(startDate,startTime).ctime(),
140 140 datetime.datetime.combine(endDate,endTime).ctime()))
141 141
142 142 return None, None
143 143
144 144 if len(dateList) > 1:
145 145 print("[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate))
146 146 else:
147 147 print("[Reading] data was found for the date %s" %(dateList[0]))
148 148
149 149 filenameList = []
150 150 datetimeList = []
151 151
152 152 #----------------------------------------------------------------------------------
153 153
154 154 for thisPath in pathList:
155 155 # thisPath = pathList[pathDict[file]]
156 156
157 157 fileList = glob.glob1(thisPath, "*%s" %ext)
158 158 fileList.sort()
159 159
160 160 for file in fileList:
161 161
162 162 filename = os.path.join(thisPath,file)
163 163
164 164 if not isFileInDateRange(filename, startDate, endDate):
165 165 continue
166 166
167 167 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
168 168
169 169 if not(thisDatetime):
170 170 continue
171 171
172 172 filenameList.append(filename)
173 173 datetimeList.append(thisDatetime)
174 174
175 175 if not(filenameList):
176 176 print("[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
177 177 return None, None
178 178
179 179 print("[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime))
180 180 print()
181 181
182 182 self.filenameList = filenameList
183 183 self.datetimeList = datetimeList
184 184
185 185 return pathList, filenameList
186 186
187 187 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
188 188
189 189 """
190 190 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
191 191
192 192 Inputs:
193 193 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
194 194
195 195 startDate : fecha inicial del rango seleccionado en formato datetime.date
196 196
197 197 endDate : fecha final del rango seleccionado en formato datetime.date
198 198
199 199 startTime : tiempo inicial del rango seleccionado en formato datetime.time
200 200
201 201 endTime : tiempo final del rango seleccionado en formato datetime.time
202 202
203 203 Return:
204 204 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
205 205 fecha especificado, de lo contrario retorna False.
206 206
207 207 Excepciones:
208 208 Si el archivo no existe o no puede ser abierto
209 209 Si la cabecera no puede ser leida.
210 210
211 211 """
212 212
213 213 try:
214 214 fp = h5py.File(filename,'r')
215 215 grp1 = fp['Data']
216 216
217 217 except IOError:
218 218 traceback.print_exc()
219 219 raise IOError("The file %s can't be opened" %(filename))
220 220 #chino rata
221 221 #In case has utctime attribute
222 222 grp2 = grp1['utctime']
223 223 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
224 224 thisUtcTime = grp2.value[0]
225 225
226 226 fp.close()
227 227
228 228 if self.timezone == 'lt':
229 229 thisUtcTime -= 5*3600
230 230
231 231 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
232 232 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
233 233 thisDate = thisDatetime.date()
234 234 thisTime = thisDatetime.time()
235 235
236 236 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
237 237 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
238 238
239 239 #General case
240 240 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
241 241 #-----------o----------------------------o-----------
242 242 # startTime endTime
243 243
244 244 if endTime >= startTime:
245 245 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
246 246 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
247 247 return thisDatetime
248 248 return None
249 249
250 250 #If endTime < startTime then endTime belongs to the next day
251 251 #<<<<<<<<<<<o o>>>>>>>>>>>
252 252 #-----------o----------------------------o-----------
253 253 # endTime startTime
254 254
255 255 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
256 256 return None
257 257
258 258 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
259 259 return None
260 260
261 261 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
262 262 return None
263 263
264 264 return thisDatetime
265 265
266 266 def __setNextFileOffline(self):
267 267
268 268 self.fileIndex += 1
269 269 idFile = self.fileIndex
270 270
271 271 if not(idFile < len(self.filenameList)):
272 272 print("No more Files")
273 273 return 0
274 274
275 275 filename = self.filenameList[idFile]
276 276
277 277 filePointer = h5py.File(filename,'r')
278 278
279 279 self.filename = filename
280 280
281 281 self.fp = filePointer
282 282
283 283 print("Setting the file: %s"%self.filename)
284 284
285 285 # self.__readMetadata()
286 286 self.__setBlockList()
287 287 self.__readData()
288 288 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
289 289 # self.nRecords = self.fp['Data'].attrs['nRecords']
290 290 self.blockIndex = 0
291 291 return 1
292 292
293 293 def __setBlockList(self):
294 294 '''
295 295 Selects the data within the times defined
296 296
297 297 self.fp
298 298 self.startTime
299 299 self.endTime
300 300
301 301 self.blockList
302 302 self.blocksPerFile
303 303
304 304 '''
305 305 fp = self.fp
306 306 startTime = self.startTime
307 307 endTime = self.endTime
308 308
309 309 grp = fp['Data']
310 310 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
311 311
312 312 #ERROOOOR
313 313 if self.timezone == 'lt':
314 314 thisUtcTime -= 5*3600
315 315
316 316 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
317 317
318 318 thisDate = thisDatetime.date()
319 319 thisTime = thisDatetime.time()
320 320
321 321 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
322 322 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
323 323
324 324 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
325 325
326 326 self.blockList = ind
327 327 self.blocksPerFile = len(ind)
328 328
329 329 return
330 330
331 331 def __readMetadata(self):
332 332 '''
333 333 Reads Metadata
334 334
335 335 self.pathMeta
336 336
337 337 self.listShapes
338 338 self.listMetaname
339 339 self.listMeta
340 340
341 341 '''
342 342
343 343 # grp = self.fp['Data']
344 344 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
345 345 #
346 346 # if pathMeta == self.pathMeta:
347 347 # return
348 348 # else:
349 349 # self.pathMeta = pathMeta
350 350 #
351 351 # filePointer = h5py.File(self.pathMeta,'r')
352 352 # groupPointer = filePointer['Metadata']
353 353
354 354 filename = self.filenameList[0]
355 355
356 356 fp = h5py.File(filename,'r')
357 357
358 358 gp = fp['Metadata']
359 359
360 360 listMetaname = []
361 361 listMetadata = []
362 362 for item in list(gp.items()):
363 363 name = item[0]
364 364
365 365 if name=='array dimensions':
366 366 table = gp[name][:]
367 367 listShapes = {}
368 368 for shapes in table:
369 369 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
370 370 else:
371 371 data = gp[name].value
372 372 listMetaname.append(name)
373 373 listMetadata.append(data)
374 374
375 375 # if name=='type':
376 376 # self.__initDataOut(data)
377 377
378 378 self.listShapes = listShapes
379 379 self.listMetaname = listMetaname
380 380 self.listMeta = listMetadata
381 381
382 382 fp.close()
383 383 return
384 384
385 385 def __readData(self):
386 386 grp = self.fp['Data']
387 387 listdataname = []
388 388 listdata = []
389 389
390 390 for item in list(grp.items()):
391 391 name = item[0]
392 392 listdataname.append(name)
393 393
394 394 array = self.__setDataArray(grp[name],self.listShapes[name])
395 395 listdata.append(array)
396 396
397 397 self.listDataname = listdataname
398 398 self.listData = listdata
399 399 return
400 400
401 401 def __setDataArray(self, dataset, shapes):
402 402
403 403 nDims = shapes[0]
404 404
405 405 nDim2 = shapes[1] #Dimension 0
406 406
407 407 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
408 408
409 409 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
410 410
411 411 mode = shapes[4] #Mode of storing
412 412
413 413 blockList = self.blockList
414 414
415 415 blocksPerFile = self.blocksPerFile
416 416
417 417 #Depending on what mode the data was stored
418 418 if mode == 0: #Divided in channels
419 419 arrayData = dataset.value.astype(numpy.float)[0][blockList]
420 420 if mode == 1: #Divided in parameter
421 421 strds = 'table'
422 422 nDatas = nDim1
423 423 newShapes = (blocksPerFile,nDim2,nDim0)
424 424 elif mode==2: #Concatenated in a table
425 425 strds = 'table0'
426 426 arrayData = dataset[strds].value
427 427 #Selecting part of the dataset
428 428 utctime = arrayData[:,0]
429 429 u, indices = numpy.unique(utctime, return_index=True)
430 430
431 431 if blockList.size != indices.size:
432 432 indMin = indices[blockList[0]]
433 433 if blockList[1] + 1 >= indices.size:
434 434 arrayData = arrayData[indMin:,:]
435 435 else:
436 436 indMax = indices[blockList[1] + 1]
437 437 arrayData = arrayData[indMin:indMax,:]
438 438 return arrayData
439 439
440 440 # One dimension
441 441 if nDims == 0:
442 442 arrayData = dataset.value.astype(numpy.float)[0][blockList]
443 443
444 444 # Two dimensions
445 445 elif nDims == 2:
446 446 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
447 447 newShapes = (blocksPerFile,nDim0)
448 448 nDatas = nDim1
449 449
450 450 for i in range(nDatas):
451 451 data = dataset[strds + str(i)].value
452 452 arrayData[:,i,:] = data[blockList,:]
453 453
454 454 # Three dimensions
455 455 else:
456 456 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
457 457 for i in range(nDatas):
458 458
459 459 data = dataset[strds + str(i)].value
460 460
461 461 for b in range(blockList.size):
462 462 arrayData[b,:,i,:] = data[:,:,blockList[b]]
463 463
464 464 return arrayData
465 465
466 466 def __setDataOut(self):
467 467 listMeta = self.listMeta
468 468 listMetaname = self.listMetaname
469 469 listDataname = self.listDataname
470 470 listData = self.listData
471 471 listShapes = self.listShapes
472 472
473 473 blockIndex = self.blockIndex
474 474 # blockList = self.blockList
475 475
476 476 for i in range(len(listMeta)):
477 477 setattr(self.dataOut,listMetaname[i],listMeta[i])
478 478
479 479 for j in range(len(listData)):
480 480 nShapes = listShapes[listDataname[j]][0]
481 481 mode = listShapes[listDataname[j]][4]
482 482 if nShapes == 1:
483 483 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
484 484 elif nShapes > 1:
485 485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
486 486 elif mode==0:
487 487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
488 488 #Mode Meteors
489 489 elif mode ==2:
490 490 selectedData = self.__selectDataMode2(listData[j], blockIndex)
491 491 setattr(self.dataOut, listDataname[j], selectedData)
492 492 return
493 493
494 494 def __selectDataMode2(self, data, blockIndex):
495 495 utctime = data[:,0]
496 496 aux, indices = numpy.unique(utctime, return_inverse=True)
497 497 selInd = numpy.where(indices == blockIndex)[0]
498 498 selData = data[selInd,:]
499 499
500 500 return selData
501 501
502 502 def getData(self):
503 503
504 504 if self.blockIndex==self.blocksPerFile:
505 505 if not( self.__setNextFileOffline() ):
506 506 self.dataOut.flagNoData = True
507 507 return 0
508 508
509 509 self.__setDataOut()
510 510 self.dataOut.flagNoData = False
511 511
512 512 self.blockIndex += 1
513 513
514 514 return
515 515
516 516 def run(self, **kwargs):
517 517
518 518 if not(self.isConfig):
519 519 self.setup(**kwargs)
520 520 # self.setObjProperties()
521 521 self.isConfig = True
522 522
523 523 self.getData()
524 524
525 525 return
526 526
527 527 @MPDecorator
528 528 class ParamWriter(Operation):
529 529 '''
530 530 HDF5 Writer, stores parameters data in HDF5 format files
531 531
532 532 path: path where the files will be stored
533 533
534 534 blocksPerFile: number of blocks that will be saved in per HDF5 format file
535 535
536 536 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
537 537
538 538 metadataList: list of attributes that will be stored as metadata
539 539
540 540 dataList: list of attributes that will be stores as data
541 541
542 542 '''
543 543
544 544
545 545 ext = ".hdf5"
546 546 optchar = "D"
547 547 metaoptchar = "M"
548 548 metaFile = None
549 549 filename = None
550 550 path = None
551 551 setFile = None
552 552 fp = None
553 553 grp = None
554 554 ds = None
555 555 firsttime = True
556 556 #Configurations
557 557 blocksPerFile = None
558 558 blockIndex = None
559 559 dataOut = None
560 560 #Data Arrays
561 561 dataList = None
562 562 metadataList = None
563 563 dsList = None #List of dictionaries with dataset properties
564 564 tableDim = None
565 565 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
566 566 currentDay = None
567 567 lastTime = None
568 568
569 569 def __init__(self):
570 570
571 571 Operation.__init__(self)
572 572 return
573 573
574 574 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
575 575 self.path = path
576 576 self.blocksPerFile = blocksPerFile
577 577 self.metadataList = metadataList
578 578 self.dataList = dataList
579 579 self.dataOut = dataOut
580 580 self.mode = mode
581 581 if self.mode is not None:
582 582 self.mode = numpy.zeros(len(self.dataList)) + mode
583 583 else:
584 584 self.mode = numpy.ones(len(self.dataList))
585 585
586 586 self.setType = setType
587 587
588 588 arrayDim = numpy.zeros((len(self.dataList),5))
589 589
590 590 #Table dimensions
591 591 dtype0 = self.dtype
592 592 tableList = []
593 593
594 594 #Dictionary and list of tables
595 595 dsList = []
596 596
597 597 for i in range(len(self.dataList)):
598 598 dsDict = {}
599 599 dataAux = getattr(self.dataOut, self.dataList[i])
600 600 dsDict['variable'] = self.dataList[i]
601 601 #--------------------- Conditionals ------------------------
602 602 #There is no data
603 603
604
605 604 if dataAux is None:
606 605
607 606 return 0
608 607
609 if type(dataAux)==float or type(dataAux)==int:
608 if isinstance(dataAux, (int, float, numpy.integer, numpy.float)):
610 609 dsDict['mode'] = 0
611 610 dsDict['nDim'] = 0
612 611 arrayDim[i,0] = 0
613 612 dsList.append(dsDict)
614 613
615 614 #Mode 2: meteors
616 615 elif self.mode[i] == 2:
617 616 dsDict['dsName'] = 'table0'
618 617 dsDict['mode'] = 2 # Mode meteors
619 618 dsDict['shape'] = dataAux.shape[-1]
620 619 dsDict['nDim'] = 0
621 620 dsDict['dsNumber'] = 1
622 621 arrayDim[i,3] = dataAux.shape[-1]
623 622 arrayDim[i,4] = self.mode[i] #Mode the data was stored
624 623 dsList.append(dsDict)
625 624
626 625 #Mode 1
627 626 else:
628 627 arrayDim0 = dataAux.shape #Data dimensions
629 628 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
630 629 arrayDim[i,4] = self.mode[i] #Mode the data was stored
631 630 strtable = 'table'
632 631 dsDict['mode'] = 1 # Mode parameters
633 632
634 633 # Three-dimension arrays
635 634 if len(arrayDim0) == 3:
636 635 arrayDim[i,1:-1] = numpy.array(arrayDim0)
637 636 nTables = int(arrayDim[i,2])
638 637 dsDict['dsNumber'] = nTables
639 638 dsDict['shape'] = arrayDim[i,2:4]
640 639 dsDict['nDim'] = 3
641 640
642 641 for j in range(nTables):
643 642 dsDict = dsDict.copy()
644 643 dsDict['dsName'] = strtable + str(j)
645 644 dsList.append(dsDict)
646 645
647 646 # Two-dimension arrays
648 647 elif len(arrayDim0) == 2:
649 648 arrayDim[i,2:-1] = numpy.array(arrayDim0)
650 649 nTables = int(arrayDim[i,2])
651 650 dsDict['dsNumber'] = nTables
652 651 dsDict['shape'] = arrayDim[i,3]
653 652 dsDict['nDim'] = 2
654 653
655 654 for j in range(nTables):
656 655 dsDict = dsDict.copy()
657 656 dsDict['dsName'] = strtable + str(j)
658 657 dsList.append(dsDict)
659 658
660 659 # One-dimension arrays
661 660 elif len(arrayDim0) == 1:
662 661 arrayDim[i,3] = arrayDim0[0]
663 662 dsDict['shape'] = arrayDim0[0]
664 663 dsDict['dsNumber'] = 1
665 664 dsDict['dsName'] = strtable + str(0)
666 665 dsDict['nDim'] = 1
667 666 dsList.append(dsDict)
668 667
669 668 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
670 669 tableList.append(table)
671 670
672 671 self.dsList = dsList
673 672 self.tableDim = numpy.array(tableList, dtype = dtype0)
674 673 self.blockIndex = 0
675 674 timeTuple = time.localtime(dataOut.utctime)
676 675 self.currentDay = timeTuple.tm_yday
677 676
678 677 def putMetadata(self):
679 678
680 679 fp = self.createMetadataFile()
681 680 self.writeMetadata(fp)
682 681 fp.close()
683 682 return
684 683
685 684 def createMetadataFile(self):
686 685 ext = self.ext
687 686 path = self.path
688 687 setFile = self.setFile
689 688
690 689 timeTuple = time.localtime(self.dataOut.utctime)
691 690
692 691 subfolder = ''
693 692 fullpath = os.path.join( path, subfolder )
694 693
695 694 if not( os.path.exists(fullpath) ):
696 695 os.mkdir(fullpath)
697 696 setFile = -1 #inicializo mi contador de seteo
698 697
699 698 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
700 699 fullpath = os.path.join( path, subfolder )
701 700
702 701 if not( os.path.exists(fullpath) ):
703 702 os.mkdir(fullpath)
704 703 setFile = -1 #inicializo mi contador de seteo
705 704
706 705 else:
707 706 filesList = os.listdir( fullpath )
708 707 filesList = sorted( filesList, key=str.lower )
709 708 if len( filesList ) > 0:
710 709 filesList = [k for k in filesList if k.startswith(self.metaoptchar)]
711 710 filen = filesList[-1]
712 711 # el filename debera tener el siguiente formato
713 712 # 0 1234 567 89A BCDE (hex)
714 713 # x YYYY DDD SSS .ext
715 714 if isNumber( filen[8:11] ):
716 715 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
717 716 else:
718 717 setFile = -1
719 718 else:
720 719 setFile = -1 #inicializo mi contador de seteo
721 720
722 721 if self.setType is None:
723 722 setFile += 1
724 723 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
725 724 timeTuple.tm_year,
726 725 timeTuple.tm_yday,
727 726 setFile,
728 727 ext )
729 728 else:
730 729 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
731 730 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
732 731 timeTuple.tm_year,
733 732 timeTuple.tm_yday,
734 733 setFile,
735 734 ext )
736 735
737 736 filename = os.path.join( path, subfolder, file )
738 737 self.metaFile = file
739 738 #Setting HDF5 File
740 739 fp = h5py.File(filename,'w')
741 740
742 741 return fp
743 742
744 743 def writeMetadata(self, fp):
745 744
746 745 grp = fp.create_group("Metadata")
747 746 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
748 747
749 748 for i in range(len(self.metadataList)):
750 749 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
751 750 return
752 751
753 752 def timeFlag(self):
754 753 currentTime = self.dataOut.utctime
755 754
756 755 if self.lastTime is None:
757 756 self.lastTime = currentTime
758 757
759 758 #Day
760 759 timeTuple = time.localtime(currentTime)
761 760 dataDay = timeTuple.tm_yday
762 761
763 762 #Time
764 763 timeDiff = currentTime - self.lastTime
765 764
766 765 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
767 766 if dataDay != self.currentDay:
768 767 self.currentDay = dataDay
769 768 return True
770 769 elif timeDiff > 3*60*60:
771 770 self.lastTime = currentTime
772 771 return True
773 772 else:
774 773 self.lastTime = currentTime
775 774 return False
776 775
777 776 def setNextFile(self):
778 777
779 778 ext = self.ext
780 779 path = self.path
781 780 setFile = self.setFile
782 781 mode = self.mode
783 782
784 783 timeTuple = time.localtime(self.dataOut.utctime)
785 784 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
786 785
787 786 fullpath = os.path.join( path, subfolder )
788 787
789 788 if os.path.exists(fullpath):
790 789 filesList = os.listdir( fullpath )
791 790 filesList = [k for k in filesList if k.startswith(self.optchar)]
792 791 if len( filesList ) > 0:
793 792 filesList = sorted( filesList, key=str.lower )
794 793 filen = filesList[-1]
795 794 # el filename debera tener el siguiente formato
796 795 # 0 1234 567 89A BCDE (hex)
797 796 # x YYYY DDD SSS .ext
798 797 if isNumber( filen[8:11] ):
799 798 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
800 799 else:
801 800 setFile = -1
802 801 else:
803 802 setFile = -1 #inicializo mi contador de seteo
804 803 else:
805 804 os.makedirs(fullpath)
806 805 setFile = -1 #inicializo mi contador de seteo
807 806
808 807 if self.setType is None:
809 808 setFile += 1
810 809 file = '%s%4.4d%3.3d%03d%s' % (self.optchar,
811 810 timeTuple.tm_year,
812 811 timeTuple.tm_yday,
813 812 setFile,
814 813 ext )
815 814 else:
816 815 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
817 816 file = '%s%4.4d%3.3d%04d%s' % (self.optchar,
818 817 timeTuple.tm_year,
819 818 timeTuple.tm_yday,
820 819 setFile,
821 820 ext )
822 821
823 822 filename = os.path.join( path, subfolder, file )
824 823
825 824 #Setting HDF5 File
826 825 fp = h5py.File(filename,'w')
827 826 #write metadata
828 827 self.writeMetadata(fp)
829 828 #Write data
830 829 grp = fp.create_group("Data")
831 830 ds = []
832 831 data = []
833 832 dsList = self.dsList
834 833 i = 0
835 834 while i < len(dsList):
836 835 dsInfo = dsList[i]
837 836 #One-dimension data
838 837 if dsInfo['mode'] == 0:
839 838 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
840 839 ds.append(ds0)
841 840 data.append([])
842 841 i += 1
843 842 continue
844 843
845 844 elif dsInfo['mode'] == 2:
846 845 grp0 = grp.create_group(dsInfo['variable'])
847 846 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
848 847 ds.append(ds0)
849 848 data.append([])
850 849 i += 1
851 850 continue
852 851
853 852 elif dsInfo['mode'] == 1:
854 853 grp0 = grp.create_group(dsInfo['variable'])
855 854
856 855 for j in range(dsInfo['dsNumber']):
857 856 dsInfo = dsList[i]
858 857 tableName = dsInfo['dsName']
859 858
860 859
861 860 if dsInfo['nDim'] == 3:
862 861 shape = dsInfo['shape'].astype(int)
863 862 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
864 863 else:
865 864 shape = int(dsInfo['shape'])
866 865 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
867 866
868 867 ds.append(ds0)
869 868 data.append([])
870 869 i += 1
871 870
872 871 fp.flush()
873 872 fp.close()
874 873
875 874 log.log('creating file: {}'.format(filename), 'Writing')
876 875 self.filename = filename
877 876 self.ds = ds
878 877 self.data = data
879 878 self.firsttime = True
880 879 self.blockIndex = 0
881 880 return
882 881
883 882 def putData(self):
884 883
885 884 if self.blockIndex == self.blocksPerFile or self.timeFlag():
886 885 self.setNextFile()
887 886
888 887 self.readBlock()
889 888 self.setBlock() #Prepare data to be written
890 889 self.writeBlock() #Write data
891 890
892 891 return
893 892
894 893 def readBlock(self):
895 894
896 895 '''
897 896 data Array configured
898 897
899 898
900 899 self.data
901 900 '''
902 901 dsList = self.dsList
903 902 ds = self.ds
904 903 #Setting HDF5 File
905 904 fp = h5py.File(self.filename,'r+')
906 905 grp = fp["Data"]
907 906 ind = 0
908 907
909 908 while ind < len(dsList):
910 909 dsInfo = dsList[ind]
911 910
912 911 if dsInfo['mode'] == 0:
913 912 ds0 = grp[dsInfo['variable']]
914 913 ds[ind] = ds0
915 914 ind += 1
916 915 else:
917 916
918 917 grp0 = grp[dsInfo['variable']]
919 918
920 919 for j in range(dsInfo['dsNumber']):
921 920 dsInfo = dsList[ind]
922 921 ds0 = grp0[dsInfo['dsName']]
923 922 ds[ind] = ds0
924 923 ind += 1
925 924
926 925 self.fp = fp
927 926 self.grp = grp
928 927 self.ds = ds
929 928
930 929 return
931 930
932 931 def setBlock(self):
933 932 '''
934 933 data Array configured
935 934
936 935
937 936 self.data
938 937 '''
939 938 #Creating Arrays
940 939 dsList = self.dsList
941 940 data = self.data
942 941 ind = 0
943 942
944 943 while ind < len(dsList):
945 944 dsInfo = dsList[ind]
946 945 dataAux = getattr(self.dataOut, dsInfo['variable'])
947 946
948 947 mode = dsInfo['mode']
949 948 nDim = dsInfo['nDim']
950 949
951 950 if mode == 0 or mode == 2 or nDim == 1:
952 951 data[ind] = dataAux
953 952 ind += 1
954 953 # elif nDim == 1:
955 954 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
956 955 # ind += 1
957 956 elif nDim == 2:
958 957 for j in range(dsInfo['dsNumber']):
959 958 data[ind] = dataAux[j,:]
960 959 ind += 1
961 960 elif nDim == 3:
962 961 for j in range(dsInfo['dsNumber']):
963 962 data[ind] = dataAux[:,j,:]
964 963 ind += 1
965 964
966 965 self.data = data
967 966 return
968 967
969 968 def writeBlock(self):
970 969 '''
971 970 Saves the block in the HDF5 file
972 971 '''
973 972 dsList = self.dsList
974 973
975 974 for i in range(len(self.ds)):
976 975 dsInfo = dsList[i]
977 976 nDim = dsInfo['nDim']
978 977 mode = dsInfo['mode']
979 978
980 979 # First time
981 980 if self.firsttime:
982 981 if type(self.data[i]) == numpy.ndarray:
983 982
984 983 if nDim == 3:
985 984 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
986 985 self.ds[i].resize(self.data[i].shape)
987 986 if mode == 2:
988 987 self.ds[i].resize(self.data[i].shape)
989 988 self.ds[i][:] = self.data[i]
990 989 else:
991 990
992 991 # From second time
993 992 # Meteors!
994 993 if mode == 2:
995 994 dataShape = self.data[i].shape
996 995 dsShape = self.ds[i].shape
997 996 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
998 997 self.ds[i][dsShape[0]:,:] = self.data[i]
999 998 # No dimension
1000 999 elif mode == 0:
1001 1000 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1002 1001 self.ds[i][0,-1] = self.data[i]
1003 1002 # One dimension
1004 1003 elif nDim == 1:
1005 1004 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1006 1005 self.ds[i][-1,:] = self.data[i]
1007 1006 # Two dimension
1008 1007 elif nDim == 2:
1009 1008 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1010 1009 self.ds[i][self.blockIndex,:] = self.data[i]
1011 1010 # Three dimensions
1012 1011 elif nDim == 3:
1013 1012 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1014 1013 self.ds[i][:,:,-1] = self.data[i]
1015 1014
1016 1015 self.firsttime = False
1017 1016 self.blockIndex += 1
1018 1017
1019 1018 #Close to save changes
1020 1019 self.fp.flush()
1021 1020 self.fp.close()
1022 1021 return
1023 1022
1024 1023 def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, setType=None):
1025 1024
1026 1025 if not(self.isConfig):
1027 1026 self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1028 1027 metadataList=metadataList, dataList=dataList, mode=mode,
1029 1028 setType=setType)
1030 1029
1031 1030 self.isConfig = True
1032 1031 self.setNextFile()
1033 1032
1034 1033 self.putData()
1035 1034 return
1036 1035 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now