##// END OF EJS Templates
schain-jc
RSS
Test commit jroIO_param.py change letter D by M
soporte -
r1216:cef0eac4db4e v2.3-devel
parent child
Show More
Side by Side Unified
Context file:
r1216:cef0eac4db4e Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1095 +1,1095
1 1 import numpy
2 2 import time
3 3 import os
4 4 import h5py
5 5 import re
6 6 import datetime
7 7
8 8 from schainpy.model.data.jrodata import *
9 9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 10 # from jroIO_base import *
11 11 from schainpy.model.io.jroIO_base import *
12 12 import schainpy
13 13
14 14
15 15 class ParamReader(ProcessingUnit):
16 16 '''
17 17 Reads HDF5 format files
18 18
19 19 path
20 20
21 21 startDate
22 22
23 23 endDate
24 24
25 25 startTime
26 26
27 27 endTime
28 28 '''
29 29
30 30 ext = ".hdf5"
31 31
32 32 optchar = "D"
33 33
34 34 timezone = None
35 35
36 36 startTime = None
37 37
38 38 endTime = None
39 39
40 40 fileIndex = None
41 41
42 42 utcList = None #To select data in the utctime list
43 43
44 44 blockList = None #List to blocks to be read from the file
45 45
46 46 blocksPerFile = None #Number of blocks to be read
47 47
48 48 blockIndex = None
49 49
50 50 path = None
51 51
52 52 #List of Files
53 53
54 54 filenameList = None
55 55
56 56 datetimeList = None
57 57
58 58 #Hdf5 File
59 59
60 60 listMetaname = None
61 61
62 62 listMeta = None
63 63
64 64 listDataname = None
65 65
66 66 listData = None
67 67
68 68 listShapes = None
69 69
70 70 fp = None
71 71
72 72 #dataOut reconstruction
73 73
74 74 dataOut = None
75 75
76 76
77 77 def __init__(self, **kwargs):
78 78 ProcessingUnit.__init__(self, **kwargs)
79 79 self.dataOut = Parameters()
80 80 return
81 81
82 82 def setup(self, **kwargs):
83 83
84 84 path = kwargs['path']
85 85 startDate = kwargs['startDate']
86 86 endDate = kwargs['endDate']
87 87 startTime = kwargs['startTime']
88 88 endTime = kwargs['endTime']
89 89 walk = kwargs['walk']
90 90 if kwargs.has_key('ext'):
91 91 ext = kwargs['ext']
92 92 else:
93 93 ext = '.hdf5'
94 94 if kwargs.has_key('timezone'):
95 95 self.timezone = kwargs['timezone']
96 96 else:
97 97 self.timezone = 'lt'
98 98
99 99 print "[Reading] Searching files in offline mode ..."
100 100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 101 startTime=startTime, endTime=endTime,
102 102 ext=ext, walk=walk)
103 103
104 104 if not(filenameList):
105 105 print "There is no files into the folder: %s"%(path)
106 106 sys.exit(-1)
107 107
108 108 self.fileIndex = -1
109 109 self.startTime = startTime
110 110 self.endTime = endTime
111 111
112 112 self.__readMetadata()
113 113
114 114 self.__setNextFileOffline()
115 115
116 116 return
117 117
118 118 def searchFilesOffLine(self,
119 119 path,
120 120 startDate=None,
121 121 endDate=None,
122 122 startTime=datetime.time(0,0,0),
123 123 endTime=datetime.time(23,59,59),
124 124 ext='.hdf5',
125 125 walk=True):
126 126
127 127 expLabel = ''
128 128 self.filenameList = []
129 129 self.datetimeList = []
130 130
131 131 pathList = []
132 132
133 133 JRODataObj = JRODataReader()
134 134 dateList, pathList = JRODataObj.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
135 135
136 136 if dateList == []:
137 137 print "[Reading] No *%s files in %s from %s to %s)"%(ext, path,
138 138 datetime.datetime.combine(startDate,startTime).ctime(),
139 139 datetime.datetime.combine(endDate,endTime).ctime())
140 140
141 141 return None, None
142 142
143 143 if len(dateList) > 1:
144 144 print "[Reading] %d days were found in date range: %s - %s" %(len(dateList), startDate, endDate)
145 145 else:
146 146 print "[Reading] data was found for the date %s" %(dateList[0])
147 147
148 148 filenameList = []
149 149 datetimeList = []
150 150
151 151 #----------------------------------------------------------------------------------
152 152
153 153 for thisPath in pathList:
154 154 # thisPath = pathList[pathDict[file]]
155 155
156 156 fileList = glob.glob1(thisPath, "*%s" %ext)
157 157 fileList.sort()
158 158
159 159 for file in fileList:
160 160
161 161 filename = os.path.join(thisPath,file)
162 162
163 163 if not isFileInDateRange(filename, startDate, endDate):
164 164 continue
165 165
166 166 thisDatetime = self.__isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
167 167
168 168 if not(thisDatetime):
169 169 continue
170 170
171 171 filenameList.append(filename)
172 172 datetimeList.append(thisDatetime)
173 173
174 174 if not(filenameList):
175 175 print "[Reading] Any file was found int time range %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
176 176 return None, None
177 177
178 178 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
179 179 print
180 180
181 181 # for i in range(len(filenameList)):
182 182 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
183 183
184 184 self.filenameList = filenameList
185 185 self.datetimeList = datetimeList
186 186
187 187 return pathList, filenameList
188 188
189 189 def __isFileInTimeRange(self,filename, startDate, endDate, startTime, endTime):
190 190
191 191 """
192 192 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
193 193
194 194 Inputs:
195 195 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
196 196
197 197 startDate : fecha inicial del rango seleccionado en formato datetime.date
198 198
199 199 endDate : fecha final del rango seleccionado en formato datetime.date
200 200
201 201 startTime : tiempo inicial del rango seleccionado en formato datetime.time
202 202
203 203 endTime : tiempo final del rango seleccionado en formato datetime.time
204 204
205 205 Return:
206 206 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
207 207 fecha especificado, de lo contrario retorna False.
208 208
209 209 Excepciones:
210 210 Si el archivo no existe o no puede ser abierto
211 211 Si la cabecera no puede ser leida.
212 212
213 213 """
214 214
215 215 try:
216 216 fp = h5py.File(filename,'r')
217 217 grp1 = fp['Data']
218 218
219 219 except IOError:
220 220 traceback.print_exc()
221 221 raise IOError, "The file %s can't be opened" %(filename)
222 222 #chino rata
223 223 #In case has utctime attribute
224 224 grp2 = grp1['utctime']
225 225 # thisUtcTime = grp2.value[0] - 5*3600 #To convert to local time
226 226 thisUtcTime = grp2.value[0]
227 227
228 228 fp.close()
229 229
230 230 if self.timezone == 'lt':
231 231 thisUtcTime -= 5*3600
232 232
233 233 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
234 234 # thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0])
235 235 thisDate = thisDatetime.date()
236 236 thisTime = thisDatetime.time()
237 237
238 238 startUtcTime = (datetime.datetime.combine(thisDate,startTime)- datetime.datetime(1970, 1, 1)).total_seconds()
239 239 endUtcTime = (datetime.datetime.combine(thisDate,endTime)- datetime.datetime(1970, 1, 1)).total_seconds()
240 240
241 241 #General case
242 242 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
243 243 #-----------o----------------------------o-----------
244 244 # startTime endTime
245 245
246 246 if endTime >= startTime:
247 247 thisUtcLog = numpy.logical_and(thisUtcTime > startUtcTime, thisUtcTime < endUtcTime)
248 248 if numpy.any(thisUtcLog): #If there is one block between the hours mentioned
249 249 return thisDatetime
250 250 return None
251 251
252 252 #If endTime < startTime then endTime belongs to the next day
253 253 #<<<<<<<<<<<o o>>>>>>>>>>>
254 254 #-----------o----------------------------o-----------
255 255 # endTime startTime
256 256
257 257 if (thisDate == startDate) and numpy.all(thisUtcTime < startUtcTime):
258 258 return None
259 259
260 260 if (thisDate == endDate) and numpy.all(thisUtcTime > endUtcTime):
261 261 return None
262 262
263 263 if numpy.all(thisUtcTime < startUtcTime) and numpy.all(thisUtcTime > endUtcTime):
264 264 return None
265 265
266 266 return thisDatetime
267 267
268 268 def __setNextFileOffline(self):
269 269
270 270 self.fileIndex += 1
271 271 idFile = self.fileIndex
272 272
273 273 if not(idFile < len(self.filenameList)):
274 274 print "No more Files"
275 275 return 0
276 276
277 277 filename = self.filenameList[idFile]
278 278
279 279 filePointer = h5py.File(filename,'r')
280 280
281 281 self.filename = filename
282 282
283 283 self.fp = filePointer
284 284
285 285 print "Setting the file: %s"%self.filename
286 286
287 287 # self.__readMetadata()
288 288 self.__setBlockList()
289 289 self.__readData()
290 290 # self.nRecords = self.fp['Data'].attrs['blocksPerFile']
291 291 # self.nRecords = self.fp['Data'].attrs['nRecords']
292 292 self.blockIndex = 0
293 293 return 1
294 294
295 295 def __setBlockList(self):
296 296 '''
297 297 Selects the data within the times defined
298 298
299 299 self.fp
300 300 self.startTime
301 301 self.endTime
302 302
303 303 self.blockList
304 304 self.blocksPerFile
305 305
306 306 '''
307 307 fp = self.fp
308 308 startTime = self.startTime
309 309 endTime = self.endTime
310 310
311 311 grp = fp['Data']
312 312 thisUtcTime = grp['utctime'].value.astype(numpy.float)[0]
313 313
314 314 #ERROOOOR
315 315 if self.timezone == 'lt':
316 316 thisUtcTime -= 5*3600
317 317
318 318 thisDatetime = datetime.datetime.fromtimestamp(thisUtcTime[0] + 5*3600)
319 319
320 320 thisDate = thisDatetime.date()
321 321 thisTime = thisDatetime.time()
322 322
323 323 startUtcTime = (datetime.datetime.combine(thisDate,startTime) - datetime.datetime(1970, 1, 1)).total_seconds()
324 324 endUtcTime = (datetime.datetime.combine(thisDate,endTime) - datetime.datetime(1970, 1, 1)).total_seconds()
325 325
326 326 ind = numpy.where(numpy.logical_and(thisUtcTime >= startUtcTime, thisUtcTime < endUtcTime))[0]
327 327
328 328 self.blockList = ind
329 329 self.blocksPerFile = len(ind)
330 330
331 331 return
332 332
333 333 def __readMetadata(self):
334 334 '''
335 335 Reads Metadata
336 336
337 337 self.pathMeta
338 338
339 339 self.listShapes
340 340 self.listMetaname
341 341 self.listMeta
342 342
343 343 '''
344 344
345 345 # grp = self.fp['Data']
346 346 # pathMeta = os.path.join(self.path, grp.attrs['metadata'])
347 347 #
348 348 # if pathMeta == self.pathMeta:
349 349 # return
350 350 # else:
351 351 # self.pathMeta = pathMeta
352 352 #
353 353 # filePointer = h5py.File(self.pathMeta,'r')
354 354 # groupPointer = filePointer['Metadata']
355 355
356 356 filename = self.filenameList[0]
357 357
358 358 fp = h5py.File(filename,'r')
359 359
360 360 gp = fp['Metadata']
361 361
362 362 listMetaname = []
363 363 listMetadata = []
364 364 for item in gp.items():
365 365 name = item[0]
366 366
367 367 if name=='array dimensions':
368 368 table = gp[name][:]
369 369 listShapes = {}
370 370 for shapes in table:
371 371 listShapes[shapes[0]] = numpy.array([shapes[1],shapes[2],shapes[3],shapes[4],shapes[5]])
372 372 else:
373 373 data = gp[name].value
374 374 listMetaname.append(name)
375 375 listMetadata.append(data)
376 376
377 377 # if name=='type':
378 378 # self.__initDataOut(data)
379 379
380 380 self.listShapes = listShapes
381 381 self.listMetaname = listMetaname
382 382 self.listMeta = listMetadata
383 383
384 384 fp.close()
385 385 return
386 386
387 387 def __readData(self):
388 388 grp = self.fp['Data']
389 389 listdataname = []
390 390 listdata = []
391 391
392 392 for item in grp.items():
393 393 name = item[0]
394 394 listdataname.append(name)
395 395
396 396 array = self.__setDataArray(grp[name],self.listShapes[name])
397 397 listdata.append(array)
398 398
399 399 self.listDataname = listdataname
400 400 self.listData = listdata
401 401 return
402 402
403 403 def __setDataArray(self, dataset, shapes):
404 404
405 405 nDims = shapes[0]
406 406
407 407 nDim2 = shapes[1] #Dimension 0
408 408
409 409 nDim1 = shapes[2] #Dimension 1, number of Points or Parameters
410 410
411 411 nDim0 = shapes[3] #Dimension 2, number of samples or ranges
412 412
413 413 mode = shapes[4] #Mode of storing
414 414
415 415 blockList = self.blockList
416 416
417 417 blocksPerFile = self.blocksPerFile
418 418
419 419 #Depending on what mode the data was stored
420 420 if mode == 0: #Divided in channels
421 421 arrayData = dataset.value.astype(numpy.float)[0][blockList]
422 422 if mode == 1: #Divided in parameter
423 423 strds = 'table'
424 424 nDatas = nDim1
425 425 newShapes = (blocksPerFile,nDim2,nDim0)
426 426 elif mode==2: #Concatenated in a table
427 427 strds = 'table0'
428 428 arrayData = dataset[strds].value
429 429 #Selecting part of the dataset
430 430 utctime = arrayData[:,0]
431 431 u, indices = numpy.unique(utctime, return_index=True)
432 432
433 433 if blockList.size != indices.size:
434 434 indMin = indices[blockList[0]]
435 435 if blockList[1] + 1 >= indices.size:
436 436 arrayData = arrayData[indMin:,:]
437 437 else:
438 438 indMax = indices[blockList[1] + 1]
439 439 arrayData = arrayData[indMin:indMax,:]
440 440 return arrayData
441 441
442 442 # One dimension
443 443 if nDims == 0:
444 444 arrayData = dataset.value.astype(numpy.float)[0][blockList]
445 445
446 446 # Two dimensions
447 447 elif nDims == 2:
448 448 arrayData = numpy.zeros((blocksPerFile,nDim1,nDim0))
449 449 newShapes = (blocksPerFile,nDim0)
450 450 nDatas = nDim1
451 451
452 452 for i in range(nDatas):
453 453 data = dataset[strds + str(i)].value
454 454 arrayData[:,i,:] = data[blockList,:]
455 455
456 456 # Three dimensions
457 457 else:
458 458 arrayData = numpy.zeros((blocksPerFile,nDim2,nDim1,nDim0))
459 459 for i in range(nDatas):
460 460
461 461 data = dataset[strds + str(i)].value
462 462
463 463 for b in range(blockList.size):
464 464 arrayData[b,:,i,:] = data[:,:,blockList[b]]
465 465
466 466 return arrayData
467 467
468 468 def __setDataOut(self):
469 469 listMeta = self.listMeta
470 470 listMetaname = self.listMetaname
471 471 listDataname = self.listDataname
472 472 listData = self.listData
473 473 listShapes = self.listShapes
474 474
475 475 blockIndex = self.blockIndex
476 476 # blockList = self.blockList
477 477
478 478 for i in range(len(listMeta)):
479 479 setattr(self.dataOut,listMetaname[i],listMeta[i])
480 480
481 481 for j in range(len(listData)):
482 482 nShapes = listShapes[listDataname[j]][0]
483 483 mode = listShapes[listDataname[j]][4]
484 484 if nShapes == 1:
485 485 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
486 486 elif nShapes > 1:
487 487 setattr(self.dataOut,listDataname[j],listData[j][blockIndex,:])
488 488 elif mode==0:
489 489 setattr(self.dataOut,listDataname[j],listData[j][blockIndex])
490 490 #Mode Meteors
491 491 elif mode ==2:
492 492 selectedData = self.__selectDataMode2(listData[j], blockIndex)
493 493 setattr(self.dataOut, listDataname[j], selectedData)
494 494 return
495 495
496 496 def __selectDataMode2(self, data, blockIndex):
497 497 utctime = data[:,0]
498 498 aux, indices = numpy.unique(utctime, return_inverse=True)
499 499 selInd = numpy.where(indices == blockIndex)[0]
500 500 selData = data[selInd,:]
501 501
502 502 return selData
503 503
504 504 def getData(self):
505 505
506 506 # if self.flagNoMoreFiles:
507 507 # self.dataOut.flagNoData = True
508 508 # print 'Process finished'
509 509 # return 0
510 510 #
511 511 if self.blockIndex==self.blocksPerFile:
512 512 if not( self.__setNextFileOffline() ):
513 513 self.dataOut.flagNoData = True
514 514 return 0
515 515
516 516 # if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
517 517 # self.dataOut.flagNoData = True
518 518 # return 0
519 519 # self.__readData()
520 520 self.__setDataOut()
521 521 self.dataOut.flagNoData = False
522 522
523 523 self.blockIndex += 1
524 524
525 525 return
526 526
527 527 def run(self, **kwargs):
528 528
529 529 if not(self.isConfig):
530 530 self.setup(**kwargs)
531 531 # self.setObjProperties()
532 532 self.isConfig = True
533 533
534 534 self.getData()
535 535
536 536 return
537 537
538 538 class ParamWriter(Operation):
539 539 '''
540 540 HDF5 Writer, stores parameters data in HDF5 format files
541 541
542 542 path: path where the files will be stored
543 543
544 544 blocksPerFile: number of blocks that will be saved in per HDF5 format file
545 545
546 546 mode: selects the data stacking mode: '0' channels, '1' parameters, '3' table (for meteors)
547 547
548 548 metadataList: list of attributes that will be stored as metadata
549 549
550 550 dataList: list of attributes that will be stores as data
551 551
552 552 '''
553 553
554 554
555 555 ext = ".hdf5"
556 556
557 557 optchar = "D"
558 558
559 559 metaoptchar = "M"
560 560
561 561 metaFile = None
562 562
563 563 filename = None
564 564
565 565 path = None
566 566
567 567 setFile = None
568 568
569 569 fp = None
570 570
571 571 grp = None
572 572
573 573 ds = None
574 574
575 575 firsttime = True
576 576
577 577 #Configurations
578 578
579 579 blocksPerFile = None
580 580
581 581 blockIndex = None
582 582
583 583 dataOut = None
584 584
585 585 #Data Arrays
586 586
587 587 dataList = None
588 588
589 589 metadataList = None
590 590
591 591 # arrayDim = None
592 592
593 593 dsList = None #List of dictionaries with dataset properties
594 594
595 595 tableDim = None
596 596
597 597 # dtype = [('arrayName', 'S20'),('nChannels', 'i'), ('nPoints', 'i'), ('nSamples', 'i'),('mode', 'b')]
598 598
599 599 dtype = [('arrayName', 'S20'),('nDimensions', 'i'), ('dim2', 'i'), ('dim1', 'i'),('dim0', 'i'),('mode', 'b')]
600 600
601 601 currentDay = None
602 602
603 603 lastTime = None
604 604
605 605 def __init__(self, **kwargs):
606 606 Operation.__init__(self, **kwargs)
607 607 self.isConfig = False
608 608 return
609 609
610 610 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
611 611 self.path = path
612 612 self.blocksPerFile = blocksPerFile
613 613 self.metadataList = metadataList
614 614 self.dataList = dataList
615 615 self.dataOut = dataOut
616 616 self.mode = mode
617 617
618 618 if self.mode is not None:
619 619 self.mode = numpy.zeros(len(self.dataList)) + mode
620 620 else:
621 621 self.mode = numpy.ones(len(self.dataList))
622 622
623 623 arrayDim = numpy.zeros((len(self.dataList),5))
624 624
625 625 #Table dimensions
626 626 dtype0 = self.dtype
627 627 tableList = []
628 628
629 629 #Dictionary and list of tables
630 630 dsList = []
631 631
632 632 for i in range(len(self.dataList)):
633 633 dsDict = {}
634 634 dataAux = getattr(self.dataOut, self.dataList[i])
635 635 dsDict['variable'] = self.dataList[i]
636 636 #--------------------- Conditionals ------------------------
637 637 #There is no data
638 638 if dataAux is None:
639 639 return 0
640 640
641 641 #Not array, just a number
642 642 #Mode 0
643 643 if type(dataAux)==float or type(dataAux)==int:
644 644 dsDict['mode'] = 0
645 645 dsDict['nDim'] = 0
646 646 arrayDim[i,0] = 0
647 647 dsList.append(dsDict)
648 648
649 649 #Mode 2: meteors
650 650 elif mode[i] == 2:
651 651 # dsDict['nDim'] = 0
652 652 dsDict['dsName'] = 'table0'
653 653 dsDict['mode'] = 2 # Mode meteors
654 654 dsDict['shape'] = dataAux.shape[-1]
655 655 dsDict['nDim'] = 0
656 656 dsDict['dsNumber'] = 1
657 657
658 658 arrayDim[i,3] = dataAux.shape[-1]
659 659 arrayDim[i,4] = mode[i] #Mode the data was stored
660 660
661 661 dsList.append(dsDict)
662 662
663 663 #Mode 1
664 664 else:
665 665 arrayDim0 = dataAux.shape #Data dimensions
666 666 arrayDim[i,0] = len(arrayDim0) #Number of array dimensions
667 667 arrayDim[i,4] = mode[i] #Mode the data was stored
668 668
669 669 strtable = 'table'
670 670 dsDict['mode'] = 1 # Mode parameters
671 671
672 672 # Three-dimension arrays
673 673 if len(arrayDim0) == 3:
674 674 arrayDim[i,1:-1] = numpy.array(arrayDim0)
675 675 nTables = int(arrayDim[i,2])
676 676 dsDict['dsNumber'] = nTables
677 677 dsDict['shape'] = arrayDim[i,2:4]
678 678 dsDict['nDim'] = 3
679 679
680 680 for j in range(nTables):
681 681 dsDict = dsDict.copy()
682 682 dsDict['dsName'] = strtable + str(j)
683 683 dsList.append(dsDict)
684 684
685 685 # Two-dimension arrays
686 686 elif len(arrayDim0) == 2:
687 687 arrayDim[i,2:-1] = numpy.array(arrayDim0)
688 688 nTables = int(arrayDim[i,2])
689 689 dsDict['dsNumber'] = nTables
690 690 dsDict['shape'] = arrayDim[i,3]
691 691 dsDict['nDim'] = 2
692 692
693 693 for j in range(nTables):
694 694 dsDict = dsDict.copy()
695 695 dsDict['dsName'] = strtable + str(j)
696 696 dsList.append(dsDict)
697 697
698 698 # One-dimension arrays
699 699 elif len(arrayDim0) == 1:
700 700 arrayDim[i,3] = arrayDim0[0]
701 701 dsDict['shape'] = arrayDim0[0]
702 702 dsDict['dsNumber'] = 1
703 703 dsDict['dsName'] = strtable + str(0)
704 704 dsDict['nDim'] = 1
705 705 dsList.append(dsDict)
706 706
707 707 table = numpy.array((self.dataList[i],) + tuple(arrayDim[i,:]),dtype = dtype0)
708 708 tableList.append(table)
709 709
710 710 # self.arrayDim = arrayDim
711 711 self.dsList = dsList
712 712 self.tableDim = numpy.array(tableList, dtype = dtype0)
713 713 self.blockIndex = 0
714 714
715 715 timeTuple = time.localtime(dataOut.utctime)
716 716 self.currentDay = timeTuple.tm_yday
717 717 return 1
718 718
719 719 def putMetadata(self):
720 720
721 721 fp = self.createMetadataFile()
722 722 self.writeMetadata(fp)
723 723 fp.close()
724 724 return
725 725
726 726 def createMetadataFile(self):
727 727 ext = self.ext
728 728 path = self.path
729 729 setFile = self.setFile
730 730
731 731 timeTuple = time.localtime(self.dataOut.utctime)
732 732
733 733 subfolder = ''
734 734 fullpath = os.path.join( path, subfolder )
735 735
736 736 if not( os.path.exists(fullpath) ):
737 737 os.mkdir(fullpath)
738 738 setFile = -1 #inicializo mi contador de seteo
739 739
740 740 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
741 741 fullpath = os.path.join( path, subfolder )
742 742
743 743 if not( os.path.exists(fullpath) ):
744 744 os.mkdir(fullpath)
745 745 setFile = -1 #inicializo mi contador de seteo
746 746
747 747 else:
748 748 filesList = os.listdir( fullpath )
749 749 filesList = sorted( filesList, key=str.lower )
750 750 if len( filesList ) > 0:
751 751 filesList = [k for k in filesList if 'M' in k]
752 752 filen = filesList[-1]
753 753 # el filename debera tener el siguiente formato
754 754 # 0 1234 567 89A BCDE (hex)
755 755 # x YYYY DDD SSS .ext
756 756 if isNumber( filen[8:11] ):
757 757 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
758 758 else:
759 759 setFile = -1
760 760 else:
761 761 setFile = -1 #inicializo mi contador de seteo
762 762
763 763 if self.setType is None:
764 764 setFile += 1
765 765 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
766 766 timeTuple.tm_year,
767 767 timeTuple.tm_yday,
768 768 setFile,
769 769 ext )
770 770 else:
771 771 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
772 772 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
773 773 timeTuple.tm_year,
774 774 timeTuple.tm_yday,
775 775 setFile,
776 776 ext )
777 777
778 778 filename = os.path.join( path, subfolder, file )
779 779 self.metaFile = file
780 780 #Setting HDF5 File
781 781 fp = h5py.File(filename,'w')
782 782
783 783 return fp
784 784
785 785 def writeMetadata(self, fp):
786 786
787 787 grp = fp.create_group("Metadata")
788 788 grp.create_dataset('array dimensions', data = self.tableDim, dtype = self.dtype)
789 789
790 790 for i in range(len(self.metadataList)):
791 791 grp.create_dataset(self.metadataList[i], data=getattr(self.dataOut, self.metadataList[i]))
792 792 return
793 793
794 794 def timeFlag(self):
795 795 currentTime = self.dataOut.utctime
796 796
797 797 if self.lastTime is None:
798 798 self.lastTime = currentTime
799 799
800 800 #Day
801 801 timeTuple = time.localtime(currentTime)
802 802 dataDay = timeTuple.tm_yday
803 803
804 804 #Time
805 805 timeDiff = currentTime - self.lastTime
806 806
807 807 #Si el dia es diferente o si la diferencia entre un dato y otro supera la hora
808 808 if dataDay != self.currentDay:
809 809 self.currentDay = dataDay
810 810 return True
811 811 elif timeDiff > 3*60*60:
812 812 self.lastTime = currentTime
813 813 return True
814 814 else:
815 815 self.lastTime = currentTime
816 816 return False
817 817
818 818 def setNextFile(self):
819 819
820 820 ext = self.ext
821 821 path = self.path
822 822 setFile = self.setFile
823 823 mode = self.mode
824 824
825 825 timeTuple = time.localtime(self.dataOut.utctime)
826 826 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
827 827
828 828 fullpath = os.path.join( path, subfolder )
829 829
830 830 if os.path.exists(fullpath):
831 831 filesList = os.listdir( fullpath )
832 filesList = [k for k in filesList if 'D' in k]
832 filesList = [k for k in filesList if 'M' in k]
833 833 if len( filesList ) > 0:
834 834 filesList = sorted( filesList, key=str.lower )
835 835 filen = filesList[-1]
836 836 # el filename debera tener el siguiente formato
837 837 # 0 1234 567 89A BCDE (hex)
838 838 # x YYYY DDD SSS .ext
839 839 if isNumber( filen[8:11] ):
840 840 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
841 841 else:
842 842 setFile = -1
843 843 else:
844 844 setFile = -1 #inicializo mi contador de seteo
845 845 else:
846 846 os.makedirs(fullpath)
847 847 setFile = -1 #inicializo mi contador de seteo
848 848
849 849 if self.setType is None:
850 850 setFile += 1
851 851 file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
852 852 timeTuple.tm_year,
853 853 timeTuple.tm_yday,
854 854 setFile,
855 855 ext )
856 856 else:
857 857 setFile = timeTuple.tm_hour*60+timeTuple.tm_min
858 858 file = '%s%4.4d%3.3d%04d%s' % (self.metaoptchar,
859 859 timeTuple.tm_year,
860 860 timeTuple.tm_yday,
861 861 setFile,
862 862 ext )
863 863
864 864 filename = os.path.join( path, subfolder, file )
865 865
866 866 #Setting HDF5 File
867 867 fp = h5py.File(filename,'w')
868 868 #write metadata
869 869 self.writeMetadata(fp)
870 870 #Write data
871 871 grp = fp.create_group("Data")
872 872 # grp.attrs['metadata'] = self.metaFile
873 873
874 874 # grp.attrs['blocksPerFile'] = 0
875 875 ds = []
876 876 data = []
877 877 dsList = self.dsList
878 878 i = 0
879 879 while i < len(dsList):
880 880 dsInfo = dsList[i]
881 881 #One-dimension data
882 882 if dsInfo['mode'] == 0:
883 883 # ds0 = grp.create_dataset(self.dataList[i], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype='S20')
884 884 ds0 = grp.create_dataset(dsInfo['variable'], (1,1), maxshape=(1,self.blocksPerFile) , chunks = True, dtype=numpy.float64)
885 885 ds.append(ds0)
886 886 data.append([])
887 887 i += 1
888 888 continue
889 889 # nDimsForDs.append(nDims[i])
890 890
891 891 elif dsInfo['mode'] == 2:
892 892 grp0 = grp.create_group(dsInfo['variable'])
893 893 ds0 = grp0.create_dataset(dsInfo['dsName'], (1,dsInfo['shape']), data = numpy.zeros((1,dsInfo['shape'])) , maxshape=(None,dsInfo['shape']), chunks=True)
894 894 ds.append(ds0)
895 895 data.append([])
896 896 i += 1
897 897 continue
898 898
899 899 elif dsInfo['mode'] == 1:
900 900 grp0 = grp.create_group(dsInfo['variable'])
901 901
902 902 for j in range(dsInfo['dsNumber']):
903 903 dsInfo = dsList[i]
904 904 tableName = dsInfo['dsName']
905 905 shape = int(dsInfo['shape'])
906 906
907 907 if dsInfo['nDim'] == 3:
908 908 ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
909 909 else:
910 910 ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
911 911
912 912 ds.append(ds0)
913 913 data.append([])
914 914 i += 1
915 915 # nDimsForDs.append(nDims[i])
916 916
917 917 fp.flush()
918 918 fp.close()
919 919
920 920 # self.nDatas = nDatas
921 921 # self.nDims = nDims
922 922 # self.nDimsForDs = nDimsForDs
923 923 #Saving variables
924 924 print 'Writing the file: %s'%filename
925 925 self.filename = filename
926 926 # self.fp = fp
927 927 # self.grp = grp
928 928 # self.grp.attrs.modify('nRecords', 1)
929 929 self.ds = ds
930 930 self.data = data
931 931 # self.setFile = setFile
932 932 self.firsttime = True
933 933 self.blockIndex = 0
934 934 return
935 935
936 936 def putData(self):
937 937
938 938 if self.blockIndex == self.blocksPerFile or self.timeFlag():
939 939 self.setNextFile()
940 940
941 941 # if not self.firsttime:
942 942 self.readBlock()
943 943 self.setBlock() #Prepare data to be written
944 944 self.writeBlock() #Write data
945 945
946 946 return
947 947
948 948 def readBlock(self):
949 949
950 950 '''
951 951 data Array configured
952 952
953 953
954 954 self.data
955 955 '''
956 956 dsList = self.dsList
957 957 ds = self.ds
958 958 #Setting HDF5 File
959 959 fp = h5py.File(self.filename,'r+')
960 960 grp = fp["Data"]
961 961 ind = 0
962 962
963 963 # grp.attrs['blocksPerFile'] = 0
964 964 while ind < len(dsList):
965 965 dsInfo = dsList[ind]
966 966
967 967 if dsInfo['mode'] == 0:
968 968 ds0 = grp[dsInfo['variable']]
969 969 ds[ind] = ds0
970 970 ind += 1
971 971 else:
972 972
973 973 grp0 = grp[dsInfo['variable']]
974 974
975 975 for j in range(dsInfo['dsNumber']):
976 976 dsInfo = dsList[ind]
977 977 ds0 = grp0[dsInfo['dsName']]
978 978 ds[ind] = ds0
979 979 ind += 1
980 980
981 981 self.fp = fp
982 982 self.grp = grp
983 983 self.ds = ds
984 984
985 985 return
986 986
987 987 def setBlock(self):
988 988 '''
989 989 data Array configured
990 990
991 991
992 992 self.data
993 993 '''
994 994 #Creating Arrays
995 995 dsList = self.dsList
996 996 data = self.data
997 997 ind = 0
998 998
999 999 while ind < len(dsList):
1000 1000 dsInfo = dsList[ind]
1001 1001 dataAux = getattr(self.dataOut, dsInfo['variable'])
1002 1002
1003 1003 mode = dsInfo['mode']
1004 1004 nDim = dsInfo['nDim']
1005 1005
1006 1006 if mode == 0 or mode == 2 or nDim == 1:
1007 1007 data[ind] = dataAux
1008 1008 ind += 1
1009 1009 # elif nDim == 1:
1010 1010 # data[ind] = numpy.reshape(dataAux,(numpy.size(dataAux),1))
1011 1011 # ind += 1
1012 1012 elif nDim == 2:
1013 1013 for j in range(dsInfo['dsNumber']):
1014 1014 data[ind] = dataAux[j,:]
1015 1015 ind += 1
1016 1016 elif nDim == 3:
1017 1017 for j in range(dsInfo['dsNumber']):
1018 1018 data[ind] = dataAux[:,j,:]
1019 1019 ind += 1
1020 1020
1021 1021 self.data = data
1022 1022 return
1023 1023
1024 1024 def writeBlock(self):
1025 1025 '''
1026 1026 Saves the block in the HDF5 file
1027 1027 '''
1028 1028 dsList = self.dsList
1029 1029
1030 1030 for i in range(len(self.ds)):
1031 1031 dsInfo = dsList[i]
1032 1032 nDim = dsInfo['nDim']
1033 1033 mode = dsInfo['mode']
1034 1034
1035 1035 # First time
1036 1036 if self.firsttime:
1037 1037 # self.ds[i].resize(self.data[i].shape)
1038 1038 # self.ds[i][self.blockIndex,:] = self.data[i]
1039 1039 if type(self.data[i]) == numpy.ndarray:
1040 1040
1041 1041 if nDim == 3:
1042 1042 self.data[i] = self.data[i].reshape((self.data[i].shape[0],self.data[i].shape[1],1))
1043 1043 self.ds[i].resize(self.data[i].shape)
1044 1044 if mode == 2:
1045 1045 self.ds[i].resize(self.data[i].shape)
1046 1046 self.ds[i][:] = self.data[i]
1047 1047 else:
1048 1048
1049 1049 # From second time
1050 1050 # Meteors!
1051 1051 if mode == 2:
1052 1052 dataShape = self.data[i].shape
1053 1053 dsShape = self.ds[i].shape
1054 1054 self.ds[i].resize((self.ds[i].shape[0] + dataShape[0],self.ds[i].shape[1]))
1055 1055 self.ds[i][dsShape[0]:,:] = self.data[i]
1056 1056 # No dimension
1057 1057 elif mode == 0:
1058 1058 self.ds[i].resize((self.ds[i].shape[0], self.ds[i].shape[1] + 1))
1059 1059 self.ds[i][0,-1] = self.data[i]
1060 1060 # One dimension
1061 1061 elif nDim == 1:
1062 1062 self.ds[i].resize((self.ds[i].shape[0] + 1, self.ds[i].shape[1]))
1063 1063 self.ds[i][-1,:] = self.data[i]
1064 1064 # Two dimension
1065 1065 elif nDim == 2:
1066 1066 self.ds[i].resize((self.ds[i].shape[0] + 1,self.ds[i].shape[1]))
1067 1067 self.ds[i][self.blockIndex,:] = self.data[i]
1068 1068 # Three dimensions
1069 1069 elif nDim == 3:
1070 1070 self.ds[i].resize((self.ds[i].shape[0],self.ds[i].shape[1],self.ds[i].shape[2]+1))
1071 1071 self.ds[i][:,:,-1] = self.data[i]
1072 1072
1073 1073 self.firsttime = False
1074 1074 self.blockIndex += 1
1075 1075
1076 1076 #Close to save changes
1077 1077 self.fp.flush()
1078 1078 self.fp.close()
1079 1079 return
1080 1080
1081 1081 def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
1082 1082
1083 1083 if not(self.isConfig):
1084 1084 flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
1085 1085 metadataList=metadataList, dataList=dataList, mode=mode, **kwargs)
1086 1086
1087 1087 if not(flagdata):
1088 1088 return
1089 1089
1090 1090 self.isConfig = True
1091 1091 # self.putMetadata()
1092 1092 self.setNextFile()
1093 1093
1094 1094 self.putData()
1095 1095 return
General Comments 0
You need to be logged in to leave comments. Login now