##// END OF EJS Templates
schain
RSS
there is a new class AMISR to read HDF5 files from that AMISR system.
Daniel Valdez -
r473:e228fb468666
parent child
Show More
Show file before | Show file after | Hide comments
@@ -643,4 +643,15 class Fits:
643 643 datatime = property(getDatatime, "I'm the 'datatime' property")
644 644 ltctime = property(getltctime, "I'm the 'ltctime' property")
645 645
646 ltctime = property(getltctime, "I'm the 'ltctime' property") No newline at end of file
646 ltctime = property(getltctime, "I'm the 'ltctime' property")
647
648 class AMISR:
649 def __init__(self):
650 self.flagNoData = True
651 self.data = None
652 self.utctime = None
653 self.type = "AMISR"
654
655 def isEmpty(self):
656
657 return self.flagNoData No newline at end of file
Show file before | Show file after | Hide comments
@@ -10,6 +10,7 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 import h5py
13 14 from xml.etree.ElementTree import Element, SubElement, ElementTree
14 15 try:
15 16 import pyfits
@@ -3418,4 +3419,407 class FitsReader(ProcessingUnit):
3418 3419 self.setup(**kwargs)
3419 3420 self.isConfig = True
3420 3421
3421 self.getData() No newline at end of file
3422 self.getData()
3423
3424
3425 class RadacHeader():
3426 def __init__(self, fp):
3427 header = 'Raw11/Data/RadacHeader'
3428 self.beamCode = fp.get(header+'/BeamCode')
3429 self.code = fp.get(header+'/Code')
3430 self.frameCount = fp.get(header+'/FrameCount')
3431 self.modeGroup = fp.get(header+'/ModeGroup')
3432 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
3433 self.pulseCount = fp.get(header+'/PulseCount')
3434 self.radacTime = fp.get(header+'/RadacTime')
3435 self.timeCount = fp.get(header+'/TimeCount')
3436 self.timeStatus = fp.get(header+'/TimeStatus')
3437
3438 self.nrecords = self.pulseCount.shape[0] #numero de bloques
3439 self.npulses = self.pulseCount.shape[1] #numero de perfiles
3440 self.nsamples = self.nsamplesPulse[0,0] #numero de alturas
3441
3442
3443 def getIndexRangeToPulse(self, idrecord=0):
3444 indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
3445 startPulseCountId = indexToZero[0][0]
3446 endPulseCountId = startPulseCountId - 1
3447 range1 = numpy.arange(startPulseCountId,self.npulses,1)
3448 range2 = numpy.arange(0,startPulseCountId,1)
3449 return range1, range2
3450
3451
3452 class AMISRReader(ProcessingUnit):
3453
3454 path = None
3455 startDate = None
3456 endDate = None
3457 startTime = None
3458 endTime = None
3459 walk = None
3460 isConfig = False
3461
3462 def __init__(self):
3463 self.set = None
3464 self.subset = None
3465 self.extension_file = '.h5'
3466 self.dtc_str = 'dtc'
3467 self.dtc_id = 0
3468 self.status = True
3469 self.isConfig = False
3470 self.dirnameList = []
3471 self.filenameList = []
3472 self.fileIndex = None
3473 self.flagNoMoreFiles = False
3474 self.flagIsNewFile = 0
3475 self.filename = ''
3476 self.amisrFilePointer = None
3477 self.radacHeaderObj = None
3478 self.dataOut = self.__createObjByDefault()
3479 self.datablock = None
3480 self.rest_datablock = None
3481 self.range = None
3482 self.idrecord_count = 0
3483 self.profileIndex = 0
3484 self.idpulse_range1 = None
3485 self.idpulse_range2 = None
3486 self.beamCodeByFrame = None
3487 self.radacTimeByFrame = None
3488 #atributos originales tal y como esta en el archivo de datos
3489 self.beamCodesFromFile = None
3490 self.radacTimeFromFile = None
3491 self.rangeFromFile = None
3492 self.dataByFrame = None
3493 self.dataset = None
3494
3495
3496 def __createObjByDefault(self):
3497
3498 dataObj = AMISR()
3499
3500 return dataObj
3501
3502 def __setParameters(self,path,startDate,endDate,startTime,endTime,walk):
3503 self.path = path
3504 self.startDate = startDate
3505 self.endDate = endDate
3506 self.startTime = startTime
3507 self.endTime = endTime
3508 self.walk = walk
3509
3510 def __checkPath(self):
3511 if os.path.exists(self.path):
3512 self.status = 1
3513 else:
3514 self.status = 0
3515 print 'Path:%s does not exists'%self.path
3516
3517 return
3518
3519 def __selDates(self, amisr_dirname_format):
3520 year = int(amisr_dirname_format[0:4])
3521 month = int(amisr_dirname_format[4:6])
3522 dom = int(amisr_dirname_format[6:8])
3523 thisDate = datetime.date(year,month,dom)
3524
3525 if (thisDate>=self.startDate and thisDate <= self.endDate):
3526 return amisr_dirname_format
3527
3528 def __findDataForDates(self):
3529
3530 import re
3531
3532 if not(self.status):
3533 return None
3534
3535 pat = '\d+.\d+'
3536 dirnameList = [re.search(pat,x).string for x in os.listdir(self.path)]
3537 dirnameList = [self.__selDates(x) for x in dirnameList]
3538 dirnameList = filter(lambda x:x!=None,dirnameList)
3539 if len(dirnameList)>0:
3540 self.status = 1
3541 self.dirnameList = dirnameList
3542 self.dirnameList.sort()
3543 else:
3544 self.status = 0
3545 return None
3546
3547 def __getTimeFromData(self):
3548 pass
3549
3550 def __filterByGlob1(self, dirName):
3551 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
3552 filterDict = {}
3553 filterDict.setdefault(dirName)
3554 filterDict[dirName] = filter_files
3555 return filterDict
3556
3557 def __getFilenameList(self, fileListInKeys, dirList):
3558 for value in fileListInKeys:
3559 dirName = value.keys()[0]
3560 for file in value[dirName]:
3561 filename = os.path.join(dirName, file)
3562 self.filenameList.append(filename)
3563
3564
3565 def __selectDataForTimes(self):
3566 #aun no esta implementado el filtro for tiempo
3567 if not(self.status):
3568 return None
3569
3570 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
3571
3572 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
3573
3574 self.__getFilenameList(fileListInKeys, dirList)
3575
3576 if len(self.filenameList)>0:
3577 self.status = 1
3578 self.filenameList.sort()
3579 else:
3580 self.status = 0
3581 return None
3582
3583
3584 def __searchFilesOffline(self,
3585 path,
3586 startDate,
3587 endDate,
3588 startTime=datetime.time(0,0,0),
3589 endTime=datetime.time(23,59,59),
3590 walk=True):
3591
3592 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
3593
3594 self.__checkPath()
3595
3596 self.__findDataForDates()
3597
3598 self.__selectDataForTimes()
3599
3600 for i in range(len(self.filenameList)):
3601 print "%s" %(self.filenameList[i])
3602
3603 return
3604
3605 def __setNextFileOffline(self):
3606 idFile = self.fileIndex
3607
3608 while (True):
3609 idFile += 1
3610 if not(idFile < len(self.filenameList)):
3611 self.flagNoMoreFiles = 1
3612 print "No more Files"
3613 return 0
3614
3615 filename = self.filenameList[idFile]
3616
3617 amisrFilePointer = h5py.File(filename,'r')
3618
3619 break
3620
3621 self.flagIsNewFile = 1
3622 self.fileIndex = idFile
3623 self.filename = filename
3624
3625 self.amisrFilePointer = amisrFilePointer
3626
3627 print "Setting the file: %s"%self.filename
3628
3629 return 1
3630
3631 def __readHeader(self):
3632 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
3633 self.flagIsNewFile = 1
3634
3635
3636
3637 def __setNextFile(self):
3638
3639 newFile = self.__setNextFileOffline()
3640
3641 if not(newFile):
3642 return 0
3643
3644 self.__readHeader()
3645
3646 self.readDataBlock()
3647
3648
3649 def setup(self,path=None,
3650 startDate=None,
3651 endDate=None,
3652 startTime=datetime.time(0,0,0),
3653 endTime=datetime.time(23,59,59),
3654 walk=True):
3655
3656 #Busqueda de archivos offline
3657 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
3658
3659 if not(self.filenameList):
3660 print "There is no files into the folder: %s"%(path)
3661
3662 sys.exit(-1)
3663
3664 self.fileIndex = -1
3665
3666 self.__setNextFile()
3667
3668 def readRanges(self):
3669 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
3670 self.rangeFromFile = dataset.values()
3671 return range
3672
3673
3674 def readRadacTime(self,idrecord, range1, range2):
3675 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
3676
3677 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
3678 #radacTimeByFrame = dataset[idrecord - 1,range1]
3679 #radacTimeByFrame = dataset[idrecord,range2]
3680
3681 return radacTimeByFrame
3682
3683 def readBeamCode(self, idrecord, range1, range2):
3684 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
3685 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
3686 self.beamCodesFromFile = dataset.value
3687
3688 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
3689 #beamcodeByFrame[range2] = dataset[idrecord, range2]
3690 beamcodeByFrame[range1] = dataset[idrecord, range1]
3691 beamcodeByFrame[range2] = dataset[idrecord, range2]
3692
3693 return beamcodeByFrame
3694
3695
3696 def __setDataByFrame(self):
3697 ndata = 2 # porque es complejo
3698 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
3699 return dataByFrame
3700
3701 def __readDataSet(self):
3702 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
3703 return dataset
3704
3705 def __setDataBlock(self,):
3706 real = self.dataByFrame[:,:,0] #asumo que 0 es real
3707 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
3708 datablock = real + imag*1j #armo el complejo
3709 return datablock
3710
3711 def readSamples_version1(self,idrecord):
3712 #estas tres primeras lineas solo se deben ejecutar una vez
3713 if self.flagIsNewFile:
3714 self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0)
3715 self.dataByFrame = self.__setDataByFrame()
3716 self.beamCodeByFrame = self.readBeamCode(idrecord, self.idpulse_range1, self.idpulse_range2)
3717 self.radacTimeByFrame = self.readRadacTime(idrecord, self.idpulse_range1, self.idpulse_range2)
3718 #reading dataset
3719 self.dataset = self.__readDataSet()
3720
3721 if idrecord == 0:
3722
3723 if len(numpy.where(self.dataByFrame!=0.0)[0]) or len(numpy.where(self.dataByFrame!=0.0)[1]) or len(numpy.where(self.dataByFrame!=0.0)[2]):
3724 #falta agregar una condicion para datos discontinuos
3725 #por defecto une los datos del record anterior
3726 self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]
3727 #timepulse
3728 self.radacTimeByFrame[self.idpulse_range2] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2]
3729 else:
3730 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]
3731
3732 self.radacTimeByFrame[self.idpulse_range1] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1]
3733
3734 datablock = self.__setDataBlock()
3735
3736 return datablock
3737
3738 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord - 1,self.idpulse_range1, :, :]
3739 self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]
3740 datablock = self.__setDataBlock()
3741 self.flagIsNewFile = 0
3742
3743 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]
3744
3745
3746 return datablock
3747
3748
3749 def readSamples(self,idrecord):
3750 if self.flagIsNewFile:
3751 self.dataByFrame = self.__setDataByFrame()
3752 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
3753
3754 #reading dataset
3755 self.dataset = self.__readDataSet()
3756
3757 self.flagIsNewFile = 0
3758 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
3759 self.dataByFrame = self.dataset[idrecord, :, :, :]
3760 datablock = self.__setDataBlock()
3761 return datablock
3762
3763
3764 def readDataBlock(self):
3765
3766 #self.datablock = self.readSamples(self.idrecord_count)
3767 self.datablock = self.readSamples(self.idrecord_count)
3768 #print 'record:', self.idrecord_count
3769
3770 self.idrecord_count += 1
3771 self.profileIndex = 0
3772
3773 if self.idrecord_count >= self.radacHeaderObj.nrecords:
3774 self.idrecord_count = 0
3775 self.flagIsNewFile = 1
3776
3777 def readNextBlock(self):
3778
3779 self.readDataBlock()
3780
3781 if self.flagIsNewFile:
3782 self.__setNextFile()
3783 pass
3784
3785 def __hasNotDataInBuffer(self):
3786 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
3787 if self.profileIndex >= self.radacHeaderObj.npulses:
3788 return 1
3789 return 0
3790
3791
3792 def getData(self):
3793
3794 if self.flagNoMoreFiles:
3795 self.dataOut.flagNoData = True
3796 print 'Process finished'
3797 return 0
3798
3799 if self.__hasNotDataInBuffer():
3800 self.readNextBlock()
3801 # if not( self.readNextBlock() ):
3802 # return 0
3803 # self.getFirstHeader()
3804
3805 if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
3806 self.dataOut.flagNoData = True
3807 return 0
3808
3809 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
3810
3811 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
3812
3813 self.dataOut.flagNoData = False
3814
3815 self.profileIndex += 1
3816
3817 return self.dataOut.data
3818
3819
3820 def run(self, **kwargs):
3821 if not(self.isConfig):
3822 self.setup(**kwargs)
3823 self.isConfig = True
3824
3825 self.getData()
General Comments 0
You need to be logged in to leave comments. Login now