##// END OF EJS Templates
schain-jc
RSS
Add setType to ParamWriter
Juan C. Espinoza -
r986:0e8ea66dee39
parent child
Show More
Show file before | Show file after | Hide comments
@@ -1,1090 +1,1105
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import datetime
7 7
8 8 from schainpy.model.data.jrodata import *
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 10 # from jroIO_base import *
11 11 from schainpy.model.io.jroIO_base import *
12 12 import schainpy
13 13
14 14
15 15 class ParamReader(ProcessingUnit):
16 16 '''
17 17 Reads HDF5 format files
18 18
19 19 path
20 20
21 21 startDate
22 22
23 23 endDate
24 24
25 25 startTime
26 26
27 27 endTime
28 28 '''
29 29
30 30 ext = ".hdf5"
31 31
32 32 optchar = "D"
33 33
34 34 timezone = None
35 35
36 36 startTime = None
37 37
38 38 endTime = None
39 39
40 40 fileIndex = None
41 41
42 42 utcList = None #To select data in the utctime list
43 43
44 44 blockList = None #List to blocks to be read from the file
45 45
46 46 blocksPerFile = None #Number of blocks to be read
47 47
48 48 blockIndex = None
49 49
50 50 path = None
51 51
52 52 #List of Files
53 53
54 54 filenameList = None
55 55
56 56 datetimeList = None
57 57
58 58 #Hdf5 File
59 59
60 60 listMetaname = None
61 61
62 62 listMeta = None
63 63
64 64 listDataname = None
65 65
66 66 listData = None
67 67
68 68 listShapes = None
69 69
70 70 fp = None
71 71
72 72 #dataOut reconstruction
73 73
74 74 dataOut = None
75 75
76 76
77 77 def __init__(self, **kwargs):
78 78 ProcessingUnit.__init__(self, **kwargs)
79 79 self.dataOut = Parameters()
80 80 return
81 81
82 82 def setup(self, **kwargs):
83 83
84 84 path = kwargs['path']
85 85 startDate = kwargs['startDate']
86 86 endDate = kwargs['endDate']
87 87 startTime = kwargs['startTime']
88 88 endTime = kwargs['endTime']
89 89 walk = kwargs['walk']
90 90 if kwargs.has_key('ext'):
91 91 ext = kwargs['ext']
92 92 else:
93 93 ext = '.hdf5'
94 94 if kwargs.has_key('timezone'):
95 95 self.timezone = kwargs['timezone']
96 96 else:
97 97 self.timezone = 'lt'
98 98
99 99 print "[Reading] Searching files in offline mode ..."
100 100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 101 startTime=startTime, endTime=endTime,
102 102 ext=ext, walk=walk)
103 103
104 104 if not(filenameList):
105 105 print "There is no files into the folder: %s"%(path)
106 106 sys.exit(-1)
107 107
108 108 self.fileIndex = -1
109 109 self.startTime = startTime
110 110 self.endTime = endTime
111 111
112 112 self.__readMetadata()
113 113
114 114 self.__setNextFileOffline()
115 115
116 116 return
117 117
118 118 def __searchFilesOffLine(self,
119 119 path,
120 120 startDate=None,
121 121 endDate=None,
122 122 startTime=datetime.time(0,0,0),
123 123 endTime=datetime.time(23,59,59),
124 124 ext='.hdf5',
125 125 walk=True):
126 126
127 127 expLabel = ''
128 128 self.filenameList = []
129 129 self.datetimeList = []
130 130
131 131 pathList = []
132 132
133 133 JRODataObj = JRODataReader()
134 134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135 135
136 136 if dateList == []:
137 137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 138 datetime.datetime.combine(startDate,startTime).ctime(),
139 139 datetime.datetime.combine(endDate,endTime).ctime())
140 140
141 141 return None, None
142 142
143 143 if len(dateList) > 1:
144 144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
145 145 else:
146 146 print "[Reading] data was found for the date %s" %(dateList[0])
147 147
148 148 filenameList = []
149 149 datetimeList = []
150 150
151 151 #----------------------------------------------------------------------------------
152 152
153 153 for thisPath in pathList:
154 154 # thisPath = pathList[pathDict[file]]
155 155
156 156 fileList = glob.glob1(thisPath, "*%s" %ext)
157 157 fileList.sort()
158 158
159 159 for file in fileList:
160 160
161 161 filename = os.path.join(thisPath,file)
162 162
163 163 if not isFileInDateRange(filename, startDate, endDate):
164 164 continue
165 165
166 166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167 167
168 168 if not(thisDatetime):
169 169 continue
170 170
171 171 filenameList.append(filename)
172 172 datetimeList.append(thisDatetime)
173 173
174 174 if not(filenameList):
175 175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
176 176 return None, None
177 177
178 178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
179 179 print
180 180
181 181 for i in range(len(filenameList)):
182 182 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183 183
184 184 self.filenameList = filenameList
185 185 self.datetimeList = datetimeList
186 186
187 187 return pathList, filenameList
188 188
189 189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190 190
191 191 """
192 192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193 193
194 194 Inputs:
195 195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196 196
197 197 startDate : fecha inicial del rango seleccionado en formato datetime.date
198 198
199 199 endDate : fecha final del rango seleccionado en formato datetime.date
200 200
201 201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202 202
203 203 endTime : tiempo final del rango seleccionado en formato datetime.time
204 204
205 205 Return:
206 206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 207 fecha especificado, de lo contrario retorna False.
208 208
209 209 Excepciones:
210 210 Si el archivo no existe o no puede ser abierto
211 211 Si la cabecera no puede ser leida.
212 212
213 213 """
214 214
215 215 try:
216 216 fp = h5py.File(filename,'r')
217 217 grp1 = fp['Data']
218 218
219 219 except IOError:
220 220 traceback.print_exc()
221 221 raise IOError, "The file %s can't be opened" %(filename)
222 222 #chino rata
223 223 #In case has utctime attribute
224 224 grp2 = grp1['utctime']
225 225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 226 thisUtcTime = grp2.value[0]
227 227
228 228 fp.close()
229 229
230 230 if self.timezone == 'lt':
231 231 thisUtcTime -= 5*3600
232 232
233 233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 235 thisDate = thisDatetime.date()
236 236 thisTime = thisDatetime.time()
237 237
238 238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240 240
241 241 #General case
242 242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 243 #-----------o----------------------------o-----------
244 244 # startTime endTime
245 245
246 246 if endTime >= startTime:
247 247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 249 return thisDatetime
250 250 return None
251 251
252 252 #If endTime < startTime then endTime belongs to the next day
253 253 #<<<<<<<<<<<o o>>>>>>>>>>>
254 254 #-----------o----------------------------o-----------
255 255 # endTime startTime
256 256
257 257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 258 return None
259 259
260 260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 261 return None
262 262
263 263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 264 return None
265 265
266 266 return thisDatetime
267 267
268 268 def __setNextFileOffline(self):
269 269
270 270 self.fileIndex += 1
271 271 idFile = self.fileIndex
272 272
273 273 if not(idFile < len(self.filenameList)):
274 274 print "No more Files"
275 275 return 0
276 276
277 277 filename = self.filenameList[idFile]
278 278
279 279 filePointer = h5py.File(filename,'r')
280 280
281 281 self.filename = filename
282 282
283 283 self.fp = filePointer
284 284
285 285 print "Setting the file: %s"%self.filename
286 286
287 287 # self.__readMetadata()
288 288 self.__setBlockList()
289 289 self.__readData()
290 290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 291 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 292 self.blockIndex = 0
293 293 return 1
294 294
295 295 def __setBlockList(self):
296 296 '''
297 297 Selects the data within the times defined
298 298
299 299 self.fp
300 300 self.startTime
301 301 self.endTime
302 302
303 303 self.blockList
304 304 self.blocksPerFile
305 305
306 306 '''
307 307 fp = self.fp
308 308 startTime = self.startTime
309 309 endTime = self.endTime
310 310
311 311 grp = fp['Data']
312 312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313 313
314 314 #ERROOOOR
315 315 if self.timezone == 'lt':
316 316 thisUtcTime -= 5*3600
317 317
318 318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319 319
320 320 thisDate = thisDatetime.date()
321 321 thisTime = thisDatetime.time()
322 322
323 323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325 325
326 326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327 327
328 328 self.blockList = ind
329 329 self.blocksPerFile = len(ind)
330 330
331 331 return
332 332
333 333 def __readMetadata(self):
334 334 '''
335 335 Reads Metadata
336 336
337 337 self.pathMeta
338 338
339 339 self.listShapes
340 340 self.listMetaname
341 341 self.listMeta
342 342
343 343 '''
344 344
345 345 # grp = self.fp['Data']
346 346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 347 #
348 348 # if pathMeta == self.pathMeta:
349 349 # return
350 350 # else:
351 351 # self.pathMeta = pathMeta
352 352 #
353 353 # filePointer = h5py.File(self.pathMeta,'r')
354 354 # groupPointer = filePointer['Metadata']
355 355
356 356 filename = self.filenameList[0]
357 357
358 358 fp = h5py.File(filename,'r')
359 359
360 360 gp = fp['Metadata']
361 361
362 362 listMetaname = []
363 363 listMetadata = []
364 364 for item in gp.items():
365 365 name = item[0]
366 366
367 367 if name=='array dimensions':
368 368 table = gp[name][:]
369 369 listShapes = {}
370 370 for shapes in table:
371 371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 372 else:
373 373 data = gp[name].value
374 374 listMetaname.append(name)
375 375 listMetadata.append(data)
376 376
377 377 # if name=='type':
378 378 # self.__initDataOut(data)
379 379
380 380 self.listShapes = listShapes
381 381 self.listMetaname = listMetaname
382 382 self.listMeta = listMetadata
383 383
384 384 fp.close()
385 385 return
386 386
387 387 def __readData(self):
388 388 grp = self.fp['Data']
389 389 listdataname = []
390 390 listdata = []
391 391
392 392 for item in grp.items():
393 393 name = item[0]
394 394 listdataname.append(name)
395 395
396 396 array = self.__setDataArray(grp[name],self.listShapes[name])
397 397 listdata.append(array)
398 398
399 399 self.listDataname = listdataname
400 400 self.listData = listdata
401 401 return
402 402
403 403 def __setDataArray(self, dataset, shapes):
404 404
405 405 nDims = shapes[0]
406 406
407 407 nDim2 = shapes[1] #Dimension 0
408 408
409 409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410 410
411 411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412 412
413 413 mode = shapes[4] #Mode of storing
414 414
415 415 blockList = self.blockList
416 416
417 417 blocksPerFile = self.blocksPerFile
418 418
419 419 #Depending on what mode the data was stored
420 420 if mode == 0: #Divided in channels
421 421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 422 if mode == 1: #Divided in parameter
423 423 strds = 'table'
424 424 nDatas = nDim1
425 425 newShapes = (blocksPerFile,nDim2,nDim0)
426 426 elif mode==2: #Concatenated in a table
427 427 strds = 'table0'
428 428 arrayData = dataset[strds].value
429 429 #Selecting part of the dataset
430 430 utctime = arrayData[:,0]
431 431 u, indices = numpy.unique(utctime, return_index=True)
432 432
433 433 if blockList.size != indices.size:
434 434 indMin = indices[blockList[0]]
435 435 if blockList[1] + 1 >= indices.size:
436 436 arrayData = arrayData[indMin:,:]
437 437 else:
438 438 indMax = indices[blockList[1] + 1]
439 439 arrayData = arrayData[indMin:indMax,:]
440 440 return arrayData
441 441
442 442 # One dimension
443 443 if nDims == 0:
444 444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445 445
446 446 # Two dimensions
447 447 elif nDims == 2:
448 448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 449 newShapes = (blocksPerFile,nDim0)
450 450 nDatas = nDim1
451 451
452 452 for i in range(nDatas):
453 453 data = dataset[strds + str(i)].value
454 454 arrayData[:,i,:] = data[blockList,:]
455 455
456 456 # Three dimensions
457 457 else:
458 458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 459 for i in range(nDatas):
460 460
461 461 data = dataset[strds + str(i)].value
462 462
463 463 for b in range(blockList.size):
464 464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465 465
466 466 return arrayData
467 467
468 468 def __setDataOut(self):
469 469 listMeta = self.listMeta
470 470 listMetaname = self.listMetaname
471 471 listDataname = self.listDataname
472 472 listData = self.listData
473 473 listShapes = self.listShapes
474 474
475 475 blockIndex = self.blockIndex
476 476 # blockList = self.blockList
477 477
478 478 for i in range(len(listMeta)):
479 479 setattr(self.dataOut,listMetaname[i],listMeta[i])
480 480
481 481 for j in range(len(listData)):
482 482 nShapes = listShapes[listDataname[j]][0]
483 483 mode = listShapes[listDataname[j]][4]
484 484 if nShapes == 1:
485 485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 486 elif nShapes > 1:
487 487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 488 elif mode==0:
489 489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 490 #Mode Meteors
491 491 elif mode ==2:
492 492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 493 setattr(self.dataOut, listDataname[j], selectedData)
494 494 return
495 495
496 496 def __selectDataMode2(self, data, blockIndex):
497 497 utctime = data[:,0]
498 498 aux, indices = numpy.unique(utctime, return_inverse=True)
499 499 selInd = numpy.where(indices == blockIndex)[0]
500 500 selData = data[selInd,:]
501 501
502 502 return selData
503 503
504 504 def getData(self):
505 505
506 506 # if self.flagNoMoreFiles:
507 507 # self.dataOut.flagNoData = True
508 508 # print 'Process finished'
509 509 # return 0
510 510 #
511 511 if self.blockIndex==self.blocksPerFile:
512 512 if not( self.__setNextFileOffline() ):
513 513 self.dataOut.flagNoData = True
514 514 return 0
515 515
516 516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 517 # self.dataOut.flagNoData = True
518 518 # return 0
519 519 # self.__readData()
520 520 self.__setDataOut()
521 521 self.dataOut.flagNoData = False
522 522
523 523 self.blockIndex += 1
524 524
525 525 return
526 526
527 527 def run(self, **kwargs):
528 528
529 529 if not(self.isConfig):
530 530 self.setup(**kwargs)
531 531 # self.setObjProperties()
532 532 self.isConfig = True
533 533
534 534 self.getData()
535 535
536 536 return
537 537
538 538 class ParamWriter(Operation):
539 539 '''
540 540 HDF5 Writer, stores parameters data in HDF5 format files
541 541
542 542 path: path where the files will be stored
543 543
544 544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545 545
546 546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547 547
548 548 metadataList: list of attributes that will be stored as metadata
549 549
550 550 dataList: list of attributes that will be stores as data
551 551
552 552 '''
553 553
554 554
555 555 ext = ".hdf5"
556 556
557 557 optchar = "D"
558 558
559 559 metaoptchar = "M"
560 560
561 561 metaFile = None
562 562
563 563 filename = None
564 564
565 565 path = None
566 566
567 567 setFile = None
568 568
569 569 fp = None
570 570
571 571 grp = None
572 572
573 573 ds = None
574 574
575 575 firsttime = True
576 576
577 577 #Configurations
578 578
579 579 blocksPerFile = None
580 580
581 581 blockIndex = None
582 582
583 583 dataOut = None
584 584
585 585 #Data Arrays
586 586
587 587 dataList = None
588 588
589 589 metadataList = None
590 590
591 591 # arrayDim = None
592 592
593 593 dsList = None #List of dictionaries with dataset properties
594 594
595 595 tableDim = None
596 596
597 597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598 598
599 599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600 600
601 601 currentDay = None
602 602
603 603 lastTime = None
604 604
605 605 def __init__(self, **kwargs):
606 606 Operation.__init__(self, **kwargs)
607 607 self.isConfig = False
608 608 return
609 609
610 610 def setup(self, dataOut, **kwargs):
611 611
612 612 self.path = kwargs['path']
613 self.setType = kwargs.get('setType', None)
613 614
614 615 if kwargs.has_key('blocksPerFile'):
615 616 self.blocksPerFile = kwargs['blocksPerFile']
616 617 else:
617 618 self.blocksPerFile = 10
618 619
619 620 self.metadataList = kwargs['metadataList']
620 621 self.dataList = kwargs['dataList']
621 622 self.dataOut = dataOut
622 623
623 624 if kwargs.has_key('mode'):
624 625 mode = kwargs['mode']
625 626
626 627 if type(mode) == int:
627 628 mode = numpy.zeros(len(self.dataList)) + mode
628 629 else:
629 630 mode = numpy.ones(len(self.dataList))
630 631
631 632 self.mode = mode
632 633
633 634 arrayDim = numpy.zeros((len(self.dataList),5))
634 635
635 636 #Table dimensions
636 637 dtype0 = self.dtype
637 638 tableList = []
638 639
639 640 #Dictionary and list of tables
640 641 dsList = []
641 642
642 643 for i in range(len(self.dataList)):
643 644 dsDict = {}
644 645 dataAux = getattr(self.dataOut, self.dataList[i])
645 646 dsDict['variable'] = self.dataList[i]
646 647 #--------------------- Conditionals ------------------------
647 648 #There is no data
648 649 if dataAux is None:
649 650 return 0
650 651
651 652 #Not array, just a number
652 653 #Mode 0
653 654 if type(dataAux)==float or type(dataAux)==int:
654 655 dsDict['mode'] = 0
655 656 dsDict['nDim'] = 0
656 657 arrayDim[i,0] = 0
657 658 dsList.append(dsDict)
658 659
659 660 #Mode 2: meteors
660 661 elif mode[i] == 2:
661 662 # dsDict['nDim'] = 0
662 663 dsDict['dsName'] = 'table0'
663 664 dsDict['mode'] = 2 # Mode meteors
664 665 dsDict['shape'] = dataAux.shape[-1]
665 666 dsDict['nDim'] = 0
666 667 dsDict['dsNumber'] = 1
667 668
668 669 arrayDim[i,3] = dataAux.shape[-1]
669 670 arrayDim[i,4] = mode[i] #Mode the data was stored
670 671
671 672 dsList.append(dsDict)
672 673
673 674 #Mode 1
674 675 else:
675 676 arrayDim0 = dataAux.shape #Data dimensions
676 677 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
677 678 arrayDim[i,4] = mode[i] #Mode the data was stored
678 679
679 680 strtable = 'table'
680 681 dsDict['mode'] = 1 # Mode parameters
681 682
682 683 # Three-dimension arrays
683 684 if len(arrayDim0) == 3:
684 685 arrayDim[i,1:-1] = numpy.array(arrayDim0)
685 686 nTables = int(arrayDim[i,2])
686 687 dsDict['dsNumber'] = nTables
687 688 dsDict['shape'] = arrayDim[i,2:4]
688 689 dsDict['nDim'] = 3
689 690
690 691 for j in range(nTables):
691 692 dsDict = dsDict.copy()
692 693 dsDict['dsName'] = strtable + str(j)
693 694 dsList.append(dsDict)
694 695
695 696 # Two-dimension arrays
696 697 elif len(arrayDim0) == 2:
697 698 arrayDim[i,2:-1] = numpy.array(arrayDim0)
698 699 nTables = int(arrayDim[i,2])
699 700 dsDict['dsNumber'] = nTables
700 701 dsDict['shape'] = arrayDim[i,3]
701 702 dsDict['nDim'] = 2
702 703
703 704 for j in range(nTables):
704 705 dsDict = dsDict.copy()
705 706 dsDict['dsName'] = strtable + str(j)
706 707 dsList.append(dsDict)
707 708
708 709 # One-dimension arrays
709 710 elif len(arrayDim0) == 1:
710 711 arrayDim[i,3] = arrayDim0[0]
711 712 dsDict['shape'] = arrayDim0[0]
712 713 dsDict['dsNumber'] = 1
713 714 dsDict['dsName'] = strtable + str(0)
714 715 dsDict['nDim'] = 1
715 716 dsList.append(dsDict)
716 717
717 718 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
718 719 tableList.append(table)
719 720
720 721 # self.arrayDim = arrayDim
721 722 self.dsList = dsList
722 723 self.tableDim = numpy.array(tableList, dtype = dtype0)
723 724 self.blockIndex = 0
724 725
725 726 timeTuple = time.localtime(dataOut.utctime)
726 727 self.currentDay = timeTuple.tm_yday
727 728 return 1
728 729
729 730 def putMetadata(self):
730 731
731 732 fp = self.createMetadataFile()
732 733 self.writeMetadata(fp)
733 734 fp.close()
734 735 return
735 736
736 737 def createMetadataFile(self):
737 738 ext = self.ext
738 739 path = self.path
739 740 setFile = self.setFile
740 741
741 742 timeTuple = time.localtime(self.dataOut.utctime)
742 743
743 744 subfolder = ''
744 745 fullpath = os.path.join( path, subfolder )
745 746
746 747 if not( os.path.exists(fullpath) ):
747 748 os.mkdir(fullpath)
748 749 setFile = -1 #inicializo mi contador de seteo
749 750
750 751 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
751 752 fullpath = os.path.join( path, subfolder )
752 753
753 754 if not( os.path.exists(fullpath) ):
754 755 os.mkdir(fullpath)
755 756 setFile = -1 #inicializo mi contador de seteo
756 757
757 758 else:
758 759 filesList = os.listdir( fullpath )
759 760 filesList = sorted( filesList, key=str.lower )
760 761 if len( filesList ) > 0:
761 762 filesList = [k for k in filesList if 'M' in k]
762 763 filen = filesList[-1]
763 764 # el filename debera tener el siguiente formato
764 765 # 0 1234 567 89A BCDE (hex)
765 766 # x YYYY DDD SSS .ext
766 767 if isNumber( filen[8:11] ):
767 768 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
768 769 else:
769 770 setFile = -1
770 771 else:
771 772 setFile = -1 #inicializo mi contador de seteo
772 773
773 setFile += 1
774
775 file = '%s%4.4d%3.3d%3.3d%s' % (self.metaoptchar,
776 timeTuple.tm_year,
777 timeTuple.tm_yday,
778 setFile,
779 ext )
774 if self.setType is None:
775 setFile += 1
776 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
777 timeTuple.tm_year,
778 timeTuple.tm_yday,
779 setFile,
780 ext )
781 else:
782 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
783 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
784 timeTuple.tm_year,
785 timeTuple.tm_yday,
786 setFile,
787 ext )
780 788
781 789 filename = os.path.join( path, subfolder, file )
782 790 self.metaFile = file
783 791 #Setting HDF5 File
784 792 fp = h5py.File(filename,'w')
785 793
786 794 return fp
787 795
788 796 def writeMetadata(self, fp):
789 797
790 798 grp = fp.create_group("Metadata")
791 799 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
792 800
793 801 for i in range(len(self.metadataList)):
794 802 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
795 803 return
796 804
797 805 def timeFlag(self):
798 806 currentTime = self.dataOut.utctime
799 807
800 808 if self.lastTime is None:
801 809 self.lastTime = currentTime
802 810
803 811 #Day
804 812 timeTuple = time.localtime(currentTime)
805 813 dataDay = timeTuple.tm_yday
806 814
807 815 #Time
808 816 timeDiff = currentTime - self.lastTime
809 817
810 818 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
811 819 if dataDay != self.currentDay:
812 820 self.currentDay = dataDay
813 821 return True
814 822 elif timeDiff > 3*60*60:
815 823 self.lastTime = currentTime
816 824 return True
817 825 else:
818 826 self.lastTime = currentTime
819 827 return False
820 828
821 829 def setNextFile(self):
822 830
823 831 ext = self.ext
824 832 path = self.path
825 833 setFile = self.setFile
826 834 mode = self.mode
827 835
828 836 timeTuple = time.localtime(self.dataOut.utctime)
829 837 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
830 838
831 839 fullpath = os.path.join( path, subfolder )
832 840
833 841 if os.path.exists(fullpath):
834 842 filesList = os.listdir( fullpath )
835 843 filesList = [k for k in filesList if 'D' in k]
836 844 if len( filesList ) > 0:
837 845 filesList = sorted( filesList, key=str.lower )
838 846 filen = filesList[-1]
839 847 # el filename debera tener el siguiente formato
840 848 # 0 1234 567 89A BCDE (hex)
841 849 # x YYYY DDD SSS .ext
842 850 if isNumber( filen[8:11] ):
843 851 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
844 852 else:
845 853 setFile = -1
846 854 else:
847 855 setFile = -1 #inicializo mi contador de seteo
848 856 else:
849 857 os.makedirs(fullpath)
850 858 setFile = -1 #inicializo mi contador de seteo
851 859
852 setFile += 1
853
854 file = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
855 timeTuple.tm_year,
856 timeTuple.tm_yday,
857 setFile,
858 ext )
860 if self.setType is None:
861 setFile += 1
862 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
863 timeTuple.tm_year,
864 timeTuple.tm_yday,
865 setFile,
866 ext )
867 else:
868 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
869 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
870 timeTuple.tm_year,
871 timeTuple.tm_yday,
872 setFile,
873 ext )
859 874
860 875 filename = os.path.join( path, subfolder, file )
861 876
862 877 #Setting HDF5 File
863 878 fp = h5py.File(filename,'w')
864 879 #write metadata
865 880 self.writeMetadata(fp)
866 881 #Write data
867 882 grp = fp.create_group("Data")
868 883 # grp.attrs['metadata'] = self.metaFile
869 884
870 885 # grp.attrs['blocksPerFile'] = 0
871 886 ds = []
872 887 data = []
873 888 dsList = self.dsList
874 889 i = 0
875 890 while i < len(dsList):
876 891 dsInfo = dsList[i]
877 892 #One-dimension data
878 893 if dsInfo['mode'] == 0:
879 894 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
880 895 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
881 896 ds.append(ds0)
882 897 data.append([])
883 898 i += 1
884 899 continue
885 900 # nDimsForDs.append(nDims[i])
886 901
887 902 elif dsInfo['mode'] == 2:
888 903 grp0 = grp.create_group(dsInfo['variable'])
889 904 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
890 905 ds.append(ds0)
891 906 data.append([])
892 907 i += 1
893 908 continue
894 909
895 910 elif dsInfo['mode'] == 1:
896 911 grp0 = grp.create_group(dsInfo['variable'])
897 912
898 913 for j in range(dsInfo['dsNumber']):
899 914 dsInfo = dsList[i]
900 915 tableName = dsInfo['dsName']
901 916 shape = int(dsInfo['shape'])
902 917
903 918 if dsInfo['nDim'] == 3:
904 919 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
905 920 else:
906 921 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
907 922
908 923 ds.append(ds0)
909 924 data.append([])
910 925 i += 1
911 926 # nDimsForDs.append(nDims[i])
912 927
913 928 fp.flush()
914 929 fp.close()
915 930
916 931 # self.nDatas = nDatas
917 932 # self.nDims = nDims
918 933 # self.nDimsForDs = nDimsForDs
919 934 #Saving variables
920 935 print 'Writing the file: %s'%filename
921 936 self.filename = filename
922 937 # self.fp = fp
923 938 # self.grp = grp
924 939 # self.grp.attrs.modify('nRecords', 1)
925 940 self.ds = ds
926 941 self.data = data
927 942 # self.setFile = setFile
928 943 self.firsttime = True
929 944 self.blockIndex = 0
930 945 return
931 946
932 947 def putData(self):
933 948
934 949 if self.blockIndex == self.blocksPerFile or self.timeFlag():
935 950 self.setNextFile()
936 951
937 952 # if not self.firsttime:
938 953 self.readBlock()
939 954 self.setBlock() #Prepare data to be written
940 955 self.writeBlock() #Write data
941 956
942 957 return
943 958
944 959 def readBlock(self):
945 960
946 961 '''
947 962 data Array configured
948 963
949 964
950 965 self.data
951 966 '''
952 967 dsList = self.dsList
953 968 ds = self.ds
954 969 #Setting HDF5 File
955 970 fp = h5py.File(self.filename,'r+')
956 971 grp = fp["Data"]
957 972 ind = 0
958 973
959 974 # grp.attrs['blocksPerFile'] = 0
960 975 while ind < len(dsList):
961 976 dsInfo = dsList[ind]
962 977
963 978 if dsInfo['mode'] == 0:
964 979 ds0 = grp[dsInfo['variable']]
965 980 ds[ind] = ds0
966 981 ind += 1
967 982 else:
968 983
969 984 grp0 = grp[dsInfo['variable']]
970 985
971 986 for j in range(dsInfo['dsNumber']):
972 987 dsInfo = dsList[ind]
973 988 ds0 = grp0[dsInfo['dsName']]
974 989 ds[ind] = ds0
975 990 ind += 1
976 991
977 992 self.fp = fp
978 993 self.grp = grp
979 994 self.ds = ds
980 995
981 996 return
982 997
983 998 def setBlock(self):
984 999 '''
985 1000 data Array configured
986 1001
987 1002
988 1003 self.data
989 1004 '''
990 1005 #Creating Arrays
991 1006 dsList = self.dsList
992 1007 data = self.data
993 1008 ind = 0
994 1009
995 1010 while ind < len(dsList):
996 1011 dsInfo = dsList[ind]
997 1012 dataAux = getattr(self.dataOut, dsInfo['variable'])
998 1013
999 1014 mode = dsInfo['mode']
1000 1015 nDim = dsInfo['nDim']
1001 1016
1002 1017 if mode == 0 or mode == 2 or nDim == 1:
1003 1018 data[ind] = dataAux
1004 1019 ind += 1
1005 1020 # elif nDim == 1:
1006 1021 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1007 1022 # ind += 1
1008 1023 elif nDim == 2:
1009 1024 for j in range(dsInfo['dsNumber']):
1010 1025 data[ind] = dataAux[j,:]
1011 1026 ind += 1
1012 1027 elif nDim == 3:
1013 1028 for j in range(dsInfo['dsNumber']):
1014 1029 data[ind] = dataAux[:,j,:]
1015 1030 ind += 1
1016 1031
1017 1032 self.data = data
1018 1033 return
1019 1034
1020 1035 def writeBlock(self):
1021 1036 '''
1022 1037 Saves the block in the HDF5 file
1023 1038 '''
1024 1039 dsList = self.dsList
1025 1040
1026 1041 for i in range(len(self.ds)):
1027 1042 dsInfo = dsList[i]
1028 1043 nDim = dsInfo['nDim']
1029 1044 mode = dsInfo['mode']
1030 1045
1031 1046 # First time
1032 1047 if self.firsttime:
1033 1048 # self.ds[i].resize(self.data[i].shape)
1034 1049 # self.ds[i][self.blockIndex,:] = self.data[i]
1035 1050 if type(self.data[i]) == numpy.ndarray:
1036 1051
1037 1052 if nDim == 3:
1038 1053 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1039 1054 self.ds[i].resize(self.data[i].shape)
1040 1055 if mode == 2:
1041 1056 self.ds[i].resize(self.data[i].shape)
1042 1057 self.ds[i][:] = self.data[i]
1043 1058 else:
1044 1059
1045 1060 # From second time
1046 1061 # Meteors!
1047 1062 if mode == 2:
1048 1063 dataShape = self.data[i].shape
1049 1064 dsShape = self.ds[i].shape
1050 1065 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1051 1066 self.ds[i][dsShape[0]:,:] = self.data[i]
1052 1067 # No dimension
1053 1068 elif mode == 0:
1054 1069 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1055 1070 self.ds[i][0,-1] = self.data[i]
1056 1071 # One dimension
1057 1072 elif nDim == 1:
1058 1073 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1059 1074 self.ds[i][-1,:] = self.data[i]
1060 1075 # Two dimension
1061 1076 elif nDim == 2:
1062 1077 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1063 1078 self.ds[i][self.blockIndex,:] = self.data[i]
1064 1079 # Three dimensions
1065 1080 elif nDim == 3:
1066 1081 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1067 1082 self.ds[i][:,:,-1] = self.data[i]
1068 1083
1069 1084 self.firsttime = False
1070 1085 self.blockIndex += 1
1071 1086
1072 1087 #Close to save changes
1073 1088 self.fp.flush()
1074 1089 self.fp.close()
1075 1090 return
1076 1091
1077 1092 def run(self, dataOut, **kwargs):
1078 1093
1079 1094 if not(self.isConfig):
1080 1095 flagdata = self.setup(dataOut, **kwargs)
1081 1096
1082 1097 if not(flagdata):
1083 1098 return
1084 1099
1085 1100 self.isConfig = True
1086 1101 # self.putMetadata()
1087 1102 self.setNextFile()
1088 1103
1089 1104 self.putData()
1090 1105 return
General Comments 0
You need to be logged in to leave comments. Login now