##// END OF EJS Templates
schain
RSS
Set default value for setType in ParamWriter
Juan C. Espinoza -
r1222:4fb9a88ae50c
parent child
Show More
Side by Side Unified
Context file:
r1222:4fb9a88ae50c Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1095 +1,1097
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import datetime
7 7
8 8 from schainpy.model.data.jrodata import *
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 10 # from jroIO_base import *
11 11 from schainpy.model.io.jroIO_base import *
12 12 import schainpy
13 13
14 14
15 15 class ParamReader(ProcessingUnit):
16 16 '''
17 17 Reads HDF5 format files
18 18
19 19 path
20 20
21 21 startDate
22 22
23 23 endDate
24 24
25 25 startTime
26 26
27 27 endTime
28 28 '''
29 29
30 30 ext = ".hdf5"
31 31
32 32 optchar = "D"
33 33
34 34 timezone = None
35 35
36 36 startTime = None
37 37
38 38 endTime = None
39 39
40 40 fileIndex = None
41 41
42 42 utcList = None #To select data in the utctime list
43 43
44 44 blockList = None #List to blocks to be read from the file
45 45
46 46 blocksPerFile = None #Number of blocks to be read
47 47
48 48 blockIndex = None
49 49
50 50 path = None
51 51
52 52 #List of Files
53 53
54 54 filenameList = None
55 55
56 56 datetimeList = None
57 57
58 58 #Hdf5 File
59 59
60 60 listMetaname = None
61 61
62 62 listMeta = None
63 63
64 64 listDataname = None
65 65
66 66 listData = None
67 67
68 68 listShapes = None
69 69
70 70 fp = None
71 71
72 72 #dataOut reconstruction
73 73
74 74 dataOut = None
75 75
76 76
77 77 def __init__(self, **kwargs):
78 78 ProcessingUnit.__init__(self, **kwargs)
79 79 self.dataOut = Parameters()
80 80 return
81 81
82 82 def setup(self, **kwargs):
83 83
84 84 path = kwargs['path']
85 85 startDate = kwargs['startDate']
86 86 endDate = kwargs['endDate']
87 87 startTime = kwargs['startTime']
88 88 endTime = kwargs['endTime']
89 89 walk = kwargs['walk']
90 90 if kwargs.has_key('ext'):
91 91 ext = kwargs['ext']
92 92 else:
93 93 ext = '.hdf5'
94 94 if kwargs.has_key('timezone'):
95 95 self.timezone = kwargs['timezone']
96 96 else:
97 97 self.timezone = 'lt'
98 98
99 99 print "[Reading] Searching files in offline mode ..."
100 100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 101 startTime=startTime, endTime=endTime,
102 102 ext=ext, walk=walk)
103 103
104 104 if not(filenameList):
105 105 print "There is no files into the folder: %s"%(path)
106 106 sys.exit(-1)
107 107
108 108 self.fileIndex = -1
109 109 self.startTime = startTime
110 110 self.endTime = endTime
111 111
112 112 self.__readMetadata()
113 113
114 114 self.__setNextFileOffline()
115 115
116 116 return
117 117
118 118 def searchFilesOffLine(self,
119 119 path,
120 120 startDate=None,
121 121 endDate=None,
122 122 startTime=datetime.time(0,0,0),
123 123 endTime=datetime.time(23,59,59),
124 124 ext='.hdf5',
125 125 walk=True):
126 126
127 127 expLabel = ''
128 128 self.filenameList = []
129 129 self.datetimeList = []
130 130
131 131 pathList = []
132 132
133 133 JRODataObj = JRODataReader()
134 134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135 135
136 136 if dateList == []:
137 137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 138 datetime.datetime.combine(startDate,startTime).ctime(),
139 139 datetime.datetime.combine(endDate,endTime).ctime())
140 140
141 141 return None, None
142 142
143 143 if len(dateList) > 1:
144 144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
145 145 else:
146 146 print "[Reading] data was found for the date %s" %(dateList[0])
147 147
148 148 filenameList = []
149 149 datetimeList = []
150 150
151 151 #----------------------------------------------------------------------------------
152 152
153 153 for thisPath in pathList:
154 154 # thisPath = pathList[pathDict[file]]
155 155
156 156 fileList = glob.glob1(thisPath, "*%s" %ext)
157 157 fileList.sort()
158 158
159 159 for file in fileList:
160 160
161 161 filename = os.path.join(thisPath,file)
162 162
163 163 if not isFileInDateRange(filename, startDate, endDate):
164 164 continue
165 165
166 166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167 167
168 168 if not(thisDatetime):
169 169 continue
170 170
171 171 filenameList.append(filename)
172 172 datetimeList.append(thisDatetime)
173 173
174 174 if not(filenameList):
175 175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
176 176 return None, None
177 177
178 178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
179 179 print
180 180
181 181 # for i in range(len(filenameList)):
182 182 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183 183
184 184 self.filenameList = filenameList
185 185 self.datetimeList = datetimeList
186 186
187 187 return pathList, filenameList
188 188
189 189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190 190
191 191 """
192 192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193 193
194 194 Inputs:
195 195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196 196
197 197 startDate : fecha inicial del rango seleccionado en formato datetime.date
198 198
199 199 endDate : fecha final del rango seleccionado en formato datetime.date
200 200
201 201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202 202
203 203 endTime : tiempo final del rango seleccionado en formato datetime.time
204 204
205 205 Return:
206 206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 207 fecha especificado, de lo contrario retorna False.
208 208
209 209 Excepciones:
210 210 Si el archivo no existe o no puede ser abierto
211 211 Si la cabecera no puede ser leida.
212 212
213 213 """
214 214
215 215 try:
216 216 fp = h5py.File(filename,'r')
217 217 grp1 = fp['Data']
218 218
219 219 except IOError:
220 220 traceback.print_exc()
221 221 raise IOError, "The file %s can't be opened" %(filename)
222 222 #chino rata
223 223 #In case has utctime attribute
224 224 grp2 = grp1['utctime']
225 225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 226 thisUtcTime = grp2.value[0]
227 227
228 228 fp.close()
229 229
230 230 if self.timezone == 'lt':
231 231 thisUtcTime -= 5*3600
232 232
233 233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 235 thisDate = thisDatetime.date()
236 236 thisTime = thisDatetime.time()
237 237
238 238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240 240
241 241 #General case
242 242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 243 #-----------o----------------------------o-----------
244 244 # startTime endTime
245 245
246 246 if endTime >= startTime:
247 247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 249 return thisDatetime
250 250 return None
251 251
252 252 #If endTime < startTime then endTime belongs to the next day
253 253 #<<<<<<<<<<<o o>>>>>>>>>>>
254 254 #-----------o----------------------------o-----------
255 255 # endTime startTime
256 256
257 257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 258 return None
259 259
260 260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 261 return None
262 262
263 263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 264 return None
265 265
266 266 return thisDatetime
267 267
268 268 def __setNextFileOffline(self):
269 269
270 270 self.fileIndex += 1
271 271 idFile = self.fileIndex
272 272
273 273 if not(idFile < len(self.filenameList)):
274 274 print "No more Files"
275 275 return 0
276 276
277 277 filename = self.filenameList[idFile]
278 278
279 279 filePointer = h5py.File(filename,'r')
280 280
281 281 self.filename = filename
282 282
283 283 self.fp = filePointer
284 284
285 285 print "Setting the file: %s"%self.filename
286 286
287 287 # self.__readMetadata()
288 288 self.__setBlockList()
289 289 self.__readData()
290 290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 291 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 292 self.blockIndex = 0
293 293 return 1
294 294
295 295 def __setBlockList(self):
296 296 '''
297 297 Selects the data within the times defined
298 298
299 299 self.fp
300 300 self.startTime
301 301 self.endTime
302 302
303 303 self.blockList
304 304 self.blocksPerFile
305 305
306 306 '''
307 307 fp = self.fp
308 308 startTime = self.startTime
309 309 endTime = self.endTime
310 310
311 311 grp = fp['Data']
312 312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313 313
314 314 #ERROOOOR
315 315 if self.timezone == 'lt':
316 316 thisUtcTime -= 5*3600
317 317
318 318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319 319
320 320 thisDate = thisDatetime.date()
321 321 thisTime = thisDatetime.time()
322 322
323 323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325 325
326 326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327 327
328 328 self.blockList = ind
329 329 self.blocksPerFile = len(ind)
330 330
331 331 return
332 332
333 333 def __readMetadata(self):
334 334 '''
335 335 Reads Metadata
336 336
337 337 self.pathMeta
338 338
339 339 self.listShapes
340 340 self.listMetaname
341 341 self.listMeta
342 342
343 343 '''
344 344
345 345 # grp = self.fp['Data']
346 346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 347 #
348 348 # if pathMeta == self.pathMeta:
349 349 # return
350 350 # else:
351 351 # self.pathMeta = pathMeta
352 352 #
353 353 # filePointer = h5py.File(self.pathMeta,'r')
354 354 # groupPointer = filePointer['Metadata']
355 355
356 356 filename = self.filenameList[0]
357 357
358 358 fp = h5py.File(filename,'r')
359 359
360 360 gp = fp['Metadata']
361 361
362 362 listMetaname = []
363 363 listMetadata = []
364 364 for item in gp.items():
365 365 name = item[0]
366 366
367 367 if name=='array dimensions':
368 368 table = gp[name][:]
369 369 listShapes = {}
370 370 for shapes in table:
371 371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 372 else:
373 373 data = gp[name].value
374 374 listMetaname.append(name)
375 375 listMetadata.append(data)
376 376
377 377 # if name=='type':
378 378 # self.__initDataOut(data)
379 379
380 380 self.listShapes = listShapes
381 381 self.listMetaname = listMetaname
382 382 self.listMeta = listMetadata
383 383
384 384 fp.close()
385 385 return
386 386
387 387 def __readData(self):
388 388 grp = self.fp['Data']
389 389 listdataname = []
390 390 listdata = []
391 391
392 392 for item in grp.items():
393 393 name = item[0]
394 394 listdataname.append(name)
395 395
396 396 array = self.__setDataArray(grp[name],self.listShapes[name])
397 397 listdata.append(array)
398 398
399 399 self.listDataname = listdataname
400 400 self.listData = listdata
401 401 return
402 402
403 403 def __setDataArray(self, dataset, shapes):
404 404
405 405 nDims = shapes[0]
406 406
407 407 nDim2 = shapes[1] #Dimension 0
408 408
409 409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410 410
411 411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412 412
413 413 mode = shapes[4] #Mode of storing
414 414
415 415 blockList = self.blockList
416 416
417 417 blocksPerFile = self.blocksPerFile
418 418
419 419 #Depending on what mode the data was stored
420 420 if mode == 0: #Divided in channels
421 421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 422 if mode == 1: #Divided in parameter
423 423 strds = 'table'
424 424 nDatas = nDim1
425 425 newShapes = (blocksPerFile,nDim2,nDim0)
426 426 elif mode==2: #Concatenated in a table
427 427 strds = 'table0'
428 428 arrayData = dataset[strds].value
429 429 #Selecting part of the dataset
430 430 utctime = arrayData[:,0]
431 431 u, indices = numpy.unique(utctime, return_index=True)
432 432
433 433 if blockList.size != indices.size:
434 434 indMin = indices[blockList[0]]
435 435 if blockList[1] + 1 >= indices.size:
436 436 arrayData = arrayData[indMin:,:]
437 437 else:
438 438 indMax = indices[blockList[1] + 1]
439 439 arrayData = arrayData[indMin:indMax,:]
440 440 return arrayData
441 441
442 442 # One dimension
443 443 if nDims == 0:
444 444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445 445
446 446 # Two dimensions
447 447 elif nDims == 2:
448 448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 449 newShapes = (blocksPerFile,nDim0)
450 450 nDatas = nDim1
451 451
452 452 for i in range(nDatas):
453 453 data = dataset[strds + str(i)].value
454 454 arrayData[:,i,:] = data[blockList,:]
455 455
456 456 # Three dimensions
457 457 else:
458 458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 459 for i in range(nDatas):
460 460
461 461 data = dataset[strds + str(i)].value
462 462
463 463 for b in range(blockList.size):
464 464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465 465
466 466 return arrayData
467 467
468 468 def __setDataOut(self):
469 469 listMeta = self.listMeta
470 470 listMetaname = self.listMetaname
471 471 listDataname = self.listDataname
472 472 listData = self.listData
473 473 listShapes = self.listShapes
474 474
475 475 blockIndex = self.blockIndex
476 476 # blockList = self.blockList
477 477
478 478 for i in range(len(listMeta)):
479 479 setattr(self.dataOut,listMetaname[i],listMeta[i])
480 480
481 481 for j in range(len(listData)):
482 482 nShapes = listShapes[listDataname[j]][0]
483 483 mode = listShapes[listDataname[j]][4]
484 484 if nShapes == 1:
485 485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 486 elif nShapes > 1:
487 487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 488 elif mode==0:
489 489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 490 #Mode Meteors
491 491 elif mode ==2:
492 492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 493 setattr(self.dataOut, listDataname[j], selectedData)
494 494 return
495 495
496 496 def __selectDataMode2(self, data, blockIndex):
497 497 utctime = data[:,0]
498 498 aux, indices = numpy.unique(utctime, return_inverse=True)
499 499 selInd = numpy.where(indices == blockIndex)[0]
500 500 selData = data[selInd,:]
501 501
502 502 return selData
503 503
504 504 def getData(self):
505 505
506 506 # if self.flagNoMoreFiles:
507 507 # self.dataOut.flagNoData = True
508 508 # print 'Process finished'
509 509 # return 0
510 510 #
511 511 if self.blockIndex==self.blocksPerFile:
512 512 if not( self.__setNextFileOffline() ):
513 513 self.dataOut.flagNoData = True
514 514 return 0
515 515
516 516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 517 # self.dataOut.flagNoData = True
518 518 # return 0
519 519 # self.__readData()
520 520 self.__setDataOut()
521 521 self.dataOut.flagNoData = False
522 522
523 523 self.blockIndex += 1
524 524
525 525 return
526 526
527 527 def run(self, **kwargs):
528 528
529 529 if not(self.isConfig):
530 530 self.setup(**kwargs)
531 531 # self.setObjProperties()
532 532 self.isConfig = True
533 533
534 534 self.getData()
535 535
536 536 return
537 537
538 538 class ParamWriter(Operation):
539 539 '''
540 540 HDF5 Writer, stores parameters data in HDF5 format files
541 541
542 542 path: path where the files will be stored
543 543
544 544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545 545
546 546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547 547
548 548 metadataList: list of attributes that will be stored as metadata
549 549
550 550 dataList: list of attributes that will be stores as data
551 551
552 552 '''
553 553
554 554
555 555 ext = ".hdf5"
556 556
557 557 optchar = "D"
558 558
559 559 metaoptchar = "M"
560 560
561 561 metaFile = None
562 562
563 563 filename = None
564 564
565 565 path = None
566 566
567 567 setFile = None
568 568
569 569 fp = None
570 570
571 571 grp = None
572 572
573 573 ds = None
574 574
575 575 firsttime = True
576 576
577 577 #Configurations
578 578
579 579 blocksPerFile = None
580 580
581 581 blockIndex = None
582 582
583 583 dataOut = None
584 584
585 585 #Data Arrays
586 586
587 587 dataList = None
588 588
589 589 metadataList = None
590 590
591 591 # arrayDim = None
592 592
593 593 dsList = None #List of dictionaries with dataset properties
594 594
595 595 tableDim = None
596 596
597 597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598 598
599 599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600 600
601 601 currentDay = None
602 602
603 603 lastTime = None
604 604
605 setType = None
606
605 607 def __init__(self, **kwargs):
606 608 Operation.__init__(self, **kwargs)
607 609 self.isConfig = False
608 610 return
609 611
610 612 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 613 self.path = path
612 614 self.blocksPerFile = blocksPerFile
613 615 self.metadataList = metadataList
614 616 self.dataList = dataList
615 617 self.dataOut = dataOut
616 618 self.mode = mode
617 619
618 620 if self.mode is not None:
619 621 self.mode = numpy.zeros(len(self.dataList)) + mode
620 622 else:
621 623 self.mode = numpy.ones(len(self.dataList))
622 624
623 625 arrayDim = numpy.zeros((len(self.dataList),5))
624 626
625 627 #Table dimensions
626 628 dtype0 = self.dtype
627 629 tableList = []
628 630
629 631 #Dictionary and list of tables
630 632 dsList = []
631 633
632 634 for i in range(len(self.dataList)):
633 635 dsDict = {}
634 636 dataAux = getattr(self.dataOut, self.dataList[i])
635 637 dsDict['variable'] = self.dataList[i]
636 638 #--------------------- Conditionals ------------------------
637 639 #There is no data
638 640 if dataAux is None:
639 641 return 0
640 642
641 643 #Not array, just a number
642 644 #Mode 0
643 645 if type(dataAux)==float or type(dataAux)==int:
644 646 dsDict['mode'] = 0
645 647 dsDict['nDim'] = 0
646 648 arrayDim[i,0] = 0
647 649 dsList.append(dsDict)
648 650
649 651 #Mode 2: meteors
650 652 elif mode[i] == 2:
651 653 # dsDict['nDim'] = 0
652 654 dsDict['dsName'] = 'table0'
653 655 dsDict['mode'] = 2 # Mode meteors
654 656 dsDict['shape'] = dataAux.shape[-1]
655 657 dsDict['nDim'] = 0
656 658 dsDict['dsNumber'] = 1
657 659
658 660 arrayDim[i,3] = dataAux.shape[-1]
659 661 arrayDim[i,4] = mode[i] #Mode the data was stored
660 662
661 663 dsList.append(dsDict)
662 664
663 665 #Mode 1
664 666 else:
665 667 arrayDim0 = dataAux.shape #Data dimensions
666 668 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
667 669 arrayDim[i,4] = mode[i] #Mode the data was stored
668 670
669 671 strtable = 'table'
670 672 dsDict['mode'] = 1 # Mode parameters
671 673
672 674 # Three-dimension arrays
673 675 if len(arrayDim0) == 3:
674 676 arrayDim[i,1:-1] = numpy.array(arrayDim0)
675 677 nTables = int(arrayDim[i,2])
676 678 dsDict['dsNumber'] = nTables
677 679 dsDict['shape'] = arrayDim[i,2:4]
678 680 dsDict['nDim'] = 3
679 681
680 682 for j in range(nTables):
681 683 dsDict = dsDict.copy()
682 684 dsDict['dsName'] = strtable + str(j)
683 685 dsList.append(dsDict)
684 686
685 687 # Two-dimension arrays
686 688 elif len(arrayDim0) == 2:
687 689 arrayDim[i,2:-1] = numpy.array(arrayDim0)
688 690 nTables = int(arrayDim[i,2])
689 691 dsDict['dsNumber'] = nTables
690 692 dsDict['shape'] = arrayDim[i,3]
691 693 dsDict['nDim'] = 2
692 694
693 695 for j in range(nTables):
694 696 dsDict = dsDict.copy()
695 697 dsDict['dsName'] = strtable + str(j)
696 698 dsList.append(dsDict)
697 699
698 700 # One-dimension arrays
699 701 elif len(arrayDim0) == 1:
700 702 arrayDim[i,3] = arrayDim0[0]
701 703 dsDict['shape'] = arrayDim0[0]
702 704 dsDict['dsNumber'] = 1
703 705 dsDict['dsName'] = strtable + str(0)
704 706 dsDict['nDim'] = 1
705 707 dsList.append(dsDict)
706 708
707 709 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
708 710 tableList.append(table)
709 711
710 712 # self.arrayDim = arrayDim
711 713 self.dsList = dsList
712 714 self.tableDim = numpy.array(tableList, dtype = dtype0)
713 715 self.blockIndex = 0
714 716
715 717 timeTuple = time.localtime(dataOut.utctime)
716 718 self.currentDay = timeTuple.tm_yday
717 719 return 1
718 720
719 721 def putMetadata(self):
720 722
721 723 fp = self.createMetadataFile()
722 724 self.writeMetadata(fp)
723 725 fp.close()
724 726 return
725 727
726 728 def createMetadataFile(self):
727 729 ext = self.ext
728 730 path = self.path
729 731 setFile = self.setFile
730 732
731 733 timeTuple = time.localtime(self.dataOut.utctime)
732 734
733 735 subfolder = ''
734 736 fullpath = os.path.join( path, subfolder )
735 737
736 738 if not( os.path.exists(fullpath) ):
737 739 os.mkdir(fullpath)
738 740 setFile = -1 #inicializo mi contador de seteo
739 741
740 742 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
741 743 fullpath = os.path.join( path, subfolder )
742 744
743 745 if not( os.path.exists(fullpath) ):
744 746 os.mkdir(fullpath)
745 747 setFile = -1 #inicializo mi contador de seteo
746 748
747 749 else:
748 750 filesList = os.listdir( fullpath )
749 751 filesList = sorted( filesList, key=str.lower )
750 752 if len( filesList ) > 0:
751 753 filesList = [k for k in filesList if 'M' in k]
752 754 filen = filesList[-1]
753 755 # el filename debera tener el siguiente formato
754 756 # 0 1234 567 89A BCDE (hex)
755 757 # x YYYY DDD SSS .ext
756 758 if isNumber( filen[8:11] ):
757 759 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
758 760 else:
759 761 setFile = -1
760 762 else:
761 763 setFile = -1 #inicializo mi contador de seteo
762 764
763 765 if self.setType is None:
764 766 setFile += 1
765 767 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
766 768 timeTuple.tm_year,
767 769 timeTuple.tm_yday,
768 770 setFile,
769 771 ext )
770 772 else:
771 773 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
772 774 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
773 775 timeTuple.tm_year,
774 776 timeTuple.tm_yday,
775 777 setFile,
776 778 ext )
777 779
778 780 filename = os.path.join( path, subfolder, file )
779 781 self.metaFile = file
780 782 #Setting HDF5 File
781 783 fp = h5py.File(filename,'w')
782 784
783 785 return fp
784 786
785 787 def writeMetadata(self, fp):
786 788
787 789 grp = fp.create_group("Metadata")
788 790 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
789 791
790 792 for i in range(len(self.metadataList)):
791 793 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
792 794 return
793 795
794 796 def timeFlag(self):
795 797 currentTime = self.dataOut.utctime
796 798
797 799 if self.lastTime is None:
798 800 self.lastTime = currentTime
799 801
800 802 #Day
801 803 timeTuple = time.localtime(currentTime)
802 804 dataDay = timeTuple.tm_yday
803 805
804 806 #Time
805 807 timeDiff = currentTime - self.lastTime
806 808
807 809 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
808 810 if dataDay != self.currentDay:
809 811 self.currentDay = dataDay
810 812 return True
811 813 elif timeDiff > 3*60*60:
812 814 self.lastTime = currentTime
813 815 return True
814 816 else:
815 817 self.lastTime = currentTime
816 818 return False
817 819
818 820 def setNextFile(self):
819 821
820 822 ext = self.ext
821 823 path = self.path
822 824 setFile = self.setFile
823 825 mode = self.mode
824 826
825 827 timeTuple = time.localtime(self.dataOut.utctime)
826 828 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
827 829
828 830 fullpath = os.path.join( path, subfolder )
829 831
830 832 if os.path.exists(fullpath):
831 833 filesList = os.listdir( fullpath )
832 834 filesList = [k for k in filesList if 'M' in k]
833 835 if len( filesList ) > 0:
834 836 filesList = sorted( filesList, key=str.lower )
835 837 filen = filesList[-1]
836 838 # el filename debera tener el siguiente formato
837 839 # 0 1234 567 89A BCDE (hex)
838 840 # x YYYY DDD SSS .ext
839 841 if isNumber( filen[8:11] ):
840 842 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
841 843 else:
842 844 setFile = -1
843 845 else:
844 846 setFile = -1 #inicializo mi contador de seteo
845 847 else:
846 848 os.makedirs(fullpath)
847 849 setFile = -1 #inicializo mi contador de seteo
848 850
849 851 if self.setType is None:
850 852 setFile += 1
851 853 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
852 854 timeTuple.tm_year,
853 855 timeTuple.tm_yday,
854 856 setFile,
855 857 ext )
856 858 else:
857 859 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
858 860 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
859 861 timeTuple.tm_year,
860 862 timeTuple.tm_yday,
861 863 setFile,
862 864 ext )
863 865
864 866 filename = os.path.join( path, subfolder, file )
865 867
866 868 #Setting HDF5 File
867 869 fp = h5py.File(filename,'w')
868 870 #write metadata
869 871 self.writeMetadata(fp)
870 872 #Write data
871 873 grp = fp.create_group("Data")
872 874 # grp.attrs['metadata'] = self.metaFile
873 875
874 876 # grp.attrs['blocksPerFile'] = 0
875 877 ds = []
876 878 data = []
877 879 dsList = self.dsList
878 880 i = 0
879 881 while i < len(dsList):
880 882 dsInfo = dsList[i]
881 883 #One-dimension data
882 884 if dsInfo['mode'] == 0:
883 885 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
884 886 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
885 887 ds.append(ds0)
886 888 data.append([])
887 889 i += 1
888 890 continue
889 891 # nDimsForDs.append(nDims[i])
890 892
891 893 elif dsInfo['mode'] == 2:
892 894 grp0 = grp.create_group(dsInfo['variable'])
893 895 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
894 896 ds.append(ds0)
895 897 data.append([])
896 898 i += 1
897 899 continue
898 900
899 901 elif dsInfo['mode'] == 1:
900 902 grp0 = grp.create_group(dsInfo['variable'])
901 903
902 904 for j in range(dsInfo['dsNumber']):
903 905 dsInfo = dsList[i]
904 906 tableName = dsInfo['dsName']
905 907 shape = int(dsInfo['shape'])
906 908
907 909 if dsInfo['nDim'] == 3:
908 910 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
909 911 else:
910 912 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
911 913
912 914 ds.append(ds0)
913 915 data.append([])
914 916 i += 1
915 917 # nDimsForDs.append(nDims[i])
916 918
917 919 fp.flush()
918 920 fp.close()
919 921
920 922 # self.nDatas = nDatas
921 923 # self.nDims = nDims
922 924 # self.nDimsForDs = nDimsForDs
923 925 #Saving variables
924 926 print 'Writing the file: %s'%filename
925 927 self.filename = filename
926 928 # self.fp = fp
927 929 # self.grp = grp
928 930 # self.grp.attrs.modify('nRecords', 1)
929 931 self.ds = ds
930 932 self.data = data
931 933 # self.setFile = setFile
932 934 self.firsttime = True
933 935 self.blockIndex = 0
934 936 return
935 937
936 938 def putData(self):
937 939
938 940 if self.blockIndex == self.blocksPerFile or self.timeFlag():
939 941 self.setNextFile()
940 942
941 943 # if not self.firsttime:
942 944 self.readBlock()
943 945 self.setBlock() #Prepare data to be written
944 946 self.writeBlock() #Write data
945 947
946 948 return
947 949
948 950 def readBlock(self):
949 951
950 952 '''
951 953 data Array configured
952 954
953 955
954 956 self.data
955 957 '''
956 958 dsList = self.dsList
957 959 ds = self.ds
958 960 #Setting HDF5 File
959 961 fp = h5py.File(self.filename,'r+')
960 962 grp = fp["Data"]
961 963 ind = 0
962 964
963 965 # grp.attrs['blocksPerFile'] = 0
964 966 while ind < len(dsList):
965 967 dsInfo = dsList[ind]
966 968
967 969 if dsInfo['mode'] == 0:
968 970 ds0 = grp[dsInfo['variable']]
969 971 ds[ind] = ds0
970 972 ind += 1
971 973 else:
972 974
973 975 grp0 = grp[dsInfo['variable']]
974 976
975 977 for j in range(dsInfo['dsNumber']):
976 978 dsInfo = dsList[ind]
977 979 ds0 = grp0[dsInfo['dsName']]
978 980 ds[ind] = ds0
979 981 ind += 1
980 982
981 983 self.fp = fp
982 984 self.grp = grp
983 985 self.ds = ds
984 986
985 987 return
986 988
987 989 def setBlock(self):
988 990 '''
989 991 data Array configured
990 992
991 993
992 994 self.data
993 995 '''
994 996 #Creating Arrays
995 997 dsList = self.dsList
996 998 data = self.data
997 999 ind = 0
998 1000
999 1001 while ind < len(dsList):
1000 1002 dsInfo = dsList[ind]
1001 1003 dataAux = getattr(self.dataOut, dsInfo['variable'])
1002 1004
1003 1005 mode = dsInfo['mode']
1004 1006 nDim = dsInfo['nDim']
1005 1007
1006 1008 if mode == 0 or mode == 2 or nDim == 1:
1007 1009 data[ind] = dataAux
1008 1010 ind += 1
1009 1011 # elif nDim == 1:
1010 1012 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1011 1013 # ind += 1
1012 1014 elif nDim == 2:
1013 1015 for j in range(dsInfo['dsNumber']):
1014 1016 data[ind] = dataAux[j,:]
1015 1017 ind += 1
1016 1018 elif nDim == 3:
1017 1019 for j in range(dsInfo['dsNumber']):
1018 1020 data[ind] = dataAux[:,j,:]
1019 1021 ind += 1
1020 1022
1021 1023 self.data = data
1022 1024 return
1023 1025
1024 1026 def writeBlock(self):
1025 1027 '''
1026 1028 Saves the block in the HDF5 file
1027 1029 '''
1028 1030 dsList = self.dsList
1029 1031
1030 1032 for i in range(len(self.ds)):
1031 1033 dsInfo = dsList[i]
1032 1034 nDim = dsInfo['nDim']
1033 1035 mode = dsInfo['mode']
1034 1036
1035 1037 # First time
1036 1038 if self.firsttime:
1037 1039 # self.ds[i].resize(self.data[i].shape)
1038 1040 # self.ds[i][self.blockIndex,:] = self.data[i]
1039 1041 if type(self.data[i]) == numpy.ndarray:
1040 1042
1041 1043 if nDim == 3:
1042 1044 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1043 1045 self.ds[i].resize(self.data[i].shape)
1044 1046 if mode == 2:
1045 1047 self.ds[i].resize(self.data[i].shape)
1046 1048 self.ds[i][:] = self.data[i]
1047 1049 else:
1048 1050
1049 1051 # From second time
1050 1052 # Meteors!
1051 1053 if mode == 2:
1052 1054 dataShape = self.data[i].shape
1053 1055 dsShape = self.ds[i].shape
1054 1056 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1055 1057 self.ds[i][dsShape[0]:,:] = self.data[i]
1056 1058 # No dimension
1057 1059 elif mode == 0:
1058 1060 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1059 1061 self.ds[i][0,-1] = self.data[i]
1060 1062 # One dimension
1061 1063 elif nDim == 1:
1062 1064 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1063 1065 self.ds[i][-1,:] = self.data[i]
1064 1066 # Two dimension
1065 1067 elif nDim == 2:
1066 1068 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1067 1069 self.ds[i][self.blockIndex,:] = self.data[i]
1068 1070 # Three dimensions
1069 1071 elif nDim == 3:
1070 1072 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1071 1073 self.ds[i][:,:,-1] = self.data[i]
1072 1074
1073 1075 self.firsttime = False
1074 1076 self.blockIndex += 1
1075 1077
1076 1078 #Close to save changes
1077 1079 self.fp.flush()
1078 1080 self.fp.close()
1079 1081 return
1080 1082
1081 1083 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1082 1084
1083 1085 if not(self.isConfig):
1084 1086 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1085 1087 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1086 1088
1087 1089 if not(flagdata):
1088 1090 return
1089 1091
1090 1092 self.isConfig = True
1091 1093 # self.putMetadata()
1092 1094 self.setNextFile()
1093 1095
1094 1096 self.putData()
1095 1097 return
General Comments 0
You need to be logged in to leave comments. Login now