##// END OF EJS Templates
schain
RSS
there is a new class AMISR to read HDF5 files from that AMISR system.
Daniel Valdez -
r473:e228fb468666
parent child
Show More
Show file before | Show file after | Hide comments
@@ -643,4 +643,15 class Fits:
643 datatime = property(getDatatime, "I'm the 'datatime' property")
643 datatime = property(getDatatime, "I'm the 'datatime' property")
644 ltctime = property(getltctime, "I'm the 'ltctime' property")
644 ltctime = property(getltctime, "I'm the 'ltctime' property")
645
645
646 ltctime = property(getltctime, "I'm the 'ltctime' property") No newline at end of file
646 ltctime = property(getltctime, "I'm the 'ltctime' property")
647
648 class AMISR:
649 def __init__(self):
650 self.flagNoData = True
651 self.data = None
652 self.utctime = None
653 self.type = "AMISR"
654
655 def isEmpty(self):
656
657 return self.flagNoData No newline at end of file
Show file before | Show file after | Hide comments
@@ -10,6 +10,7 import time
10 import numpy
10 import numpy
11 import fnmatch
11 import fnmatch
12 import time, datetime
12 import time, datetime
13 import h5py
13 from xml.etree.ElementTree import Element, SubElement, ElementTree
14 from xml.etree.ElementTree import Element, SubElement, ElementTree
14 try:
15 try:
15 import pyfits
16 import pyfits
@@ -3418,4 +3419,407 class FitsReader(ProcessingUnit):
3418 self.setup(**kwargs)
3419 self.setup(**kwargs)
3419 self.isConfig = True
3420 self.isConfig = True
3420
3421
3421 self.getData() No newline at end of file
3422 self.getData()
3423
3424
3425 class RadacHeader():
3426 def __init__(self, fp):
3427 header = 'Raw11/Data/RadacHeader'
3428 self.beamCode = fp.get(header+'/BeamCode')
3429 self.code = fp.get(header+'/Code')
3430 self.frameCount = fp.get(header+'/FrameCount')
3431 self.modeGroup = fp.get(header+'/ModeGroup')
3432 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
3433 self.pulseCount = fp.get(header+'/PulseCount')
3434 self.radacTime = fp.get(header+'/RadacTime')
3435 self.timeCount = fp.get(header+'/TimeCount')
3436 self.timeStatus = fp.get(header+'/TimeStatus')
3437
3438 self.nrecords = self.pulseCount.shape[0] #numero de bloques
3439 self.npulses = self.pulseCount.shape[1] #numero de perfiles
3440 self.nsamples = self.nsamplesPulse[0,0] #numero de alturas
3441
3442
3443 def getIndexRangeToPulse(self, idrecord=0):
3444 indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
3445 startPulseCountId = indexToZero[0][0]
3446 endPulseCountId = startPulseCountId - 1
3447 range1 = numpy.arange(startPulseCountId,self.npulses,1)
3448 range2 = numpy.arange(0,startPulseCountId,1)
3449 return range1, range2
3450
3451
3452 class AMISRReader(ProcessingUnit):
3453
3454 path = None
3455 startDate = None
3456 endDate = None
3457 startTime = None
3458 endTime = None
3459 walk = None
3460 isConfig = False
3461
3462 def __init__(self):
3463 self.set = None
3464 self.subset = None
3465 self.extension_file = '.h5'
3466 self.dtc_str = 'dtc'
3467 self.dtc_id = 0
3468 self.status = True
3469 self.isConfig = False
3470 self.dirnameList = []
3471 self.filenameList = []
3472 self.fileIndex = None
3473 self.flagNoMoreFiles = False
3474 self.flagIsNewFile = 0
3475 self.filename = ''
3476 self.amisrFilePointer = None
3477 self.radacHeaderObj = None
3478 self.dataOut = self.__createObjByDefault()
3479 self.datablock = None
3480 self.rest_datablock = None
3481 self.range = None
3482 self.idrecord_count = 0
3483 self.profileIndex = 0
3484 self.idpulse_range1 = None
3485 self.idpulse_range2 = None
3486 self.beamCodeByFrame = None
3487 self.radacTimeByFrame = None
3488 #atributos originales tal y como esta en el archivo de datos
3489 self.beamCodesFromFile = None
3490 self.radacTimeFromFile = None
3491 self.rangeFromFile = None
3492 self.dataByFrame = None
3493 self.dataset = None
3494
3495
3496 def __createObjByDefault(self):
3497
3498 dataObj = AMISR()
3499
3500 return dataObj
3501
3502 def __setParameters(self,path,startDate,endDate,startTime,endTime,walk):
3503 self.path = path
3504 self.startDate = startDate
3505 self.endDate = endDate
3506 self.startTime = startTime
3507 self.endTime = endTime
3508 self.walk = walk
3509
3510 def __checkPath(self):
3511 if os.path.exists(self.path):
3512 self.status = 1
3513 else:
3514 self.status = 0
3515 print 'Path:%s does not exists'%self.path
3516
3517 return
3518
3519 def __selDates(self, amisr_dirname_format):
3520 year = int(amisr_dirname_format[0:4])
3521 month = int(amisr_dirname_format[4:6])
3522 dom = int(amisr_dirname_format[6:8])
3523 thisDate = datetime.date(year,month,dom)
3524
3525 if (thisDate>=self.startDate and thisDate <= self.endDate):
3526 return amisr_dirname_format
3527
3528 def __findDataForDates(self):
3529
3530 import re
3531
3532 if not(self.status):
3533 return None
3534
3535 pat = '\d+.\d+'
3536 dirnameList = [re.search(pat,x).string for x in os.listdir(self.path)]
3537 dirnameList = [self.__selDates(x) for x in dirnameList]
3538 dirnameList = filter(lambda x:x!=None,dirnameList)
3539 if len(dirnameList)>0:
3540 self.status = 1
3541 self.dirnameList = dirnameList
3542 self.dirnameList.sort()
3543 else:
3544 self.status = 0
3545 return None
3546
3547 def __getTimeFromData(self):
3548 pass
3549
3550 def __filterByGlob1(self, dirName):
3551 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
3552 filterDict = {}
3553 filterDict.setdefault(dirName)
3554 filterDict[dirName] = filter_files
3555 return filterDict
3556
3557 def __getFilenameList(self, fileListInKeys, dirList):
3558 for value in fileListInKeys:
3559 dirName = value.keys()[0]
3560 for file in value[dirName]:
3561 filename = os.path.join(dirName, file)
3562 self.filenameList.append(filename)
3563
3564
3565 def __selectDataForTimes(self):
3566 #aun no esta implementado el filtro for tiempo
3567 if not(self.status):
3568 return None
3569
3570 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
3571
3572 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
3573
3574 self.__getFilenameList(fileListInKeys, dirList)
3575
3576 if len(self.filenameList)>0:
3577 self.status = 1
3578 self.filenameList.sort()
3579 else:
3580 self.status = 0
3581 return None
3582
3583
3584 def __searchFilesOffline(self,
3585 path,
3586 startDate,
3587 endDate,
3588 startTime=datetime.time(0,0,0),
3589 endTime=datetime.time(23,59,59),
3590 walk=True):
3591
3592 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
3593
3594 self.__checkPath()
3595
3596 self.__findDataForDates()
3597
3598 self.__selectDataForTimes()
3599
3600 for i in range(len(self.filenameList)):
3601 print "%s" %(self.filenameList[i])
3602
3603 return
3604
3605 def __setNextFileOffline(self):
3606 idFile = self.fileIndex
3607
3608 while (True):
3609 idFile += 1
3610 if not(idFile < len(self.filenameList)):
3611 self.flagNoMoreFiles = 1
3612 print "No more Files"
3613 return 0
3614
3615 filename = self.filenameList[idFile]
3616
3617 amisrFilePointer = h5py.File(filename,'r')
3618
3619 break
3620
3621 self.flagIsNewFile = 1
3622 self.fileIndex = idFile
3623 self.filename = filename
3624
3625 self.amisrFilePointer = amisrFilePointer
3626
3627 print "Setting the file: %s"%self.filename
3628
3629 return 1
3630
3631 def __readHeader(self):
3632 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
3633 self.flagIsNewFile = 1
3634
3635
3636
3637 def __setNextFile(self):
3638
3639 newFile = self.__setNextFileOffline()
3640
3641 if not(newFile):
3642 return 0
3643
3644 self.__readHeader()
3645
3646 self.readDataBlock()
3647
3648
3649 def setup(self,path=None,
3650 startDate=None,
3651 endDate=None,
3652 startTime=datetime.time(0,0,0),
3653 endTime=datetime.time(23,59,59),
3654 walk=True):
3655
3656 #Busqueda de archivos offline
3657 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
3658
3659 if not(self.filenameList):
3660 print "There is no files into the folder: %s"%(path)
3661
3662 sys.exit(-1)
3663
3664 self.fileIndex = -1
3665
3666 self.__setNextFile()
3667
3668 def readRanges(self):
3669 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
3670 self.rangeFromFile = dataset.values()
3671 return range
3672
3673
3674 def readRadacTime(self,idrecord, range1, range2):
3675 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
3676
3677 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
3678 #radacTimeByFrame = dataset[idrecord - 1,range1]
3679 #radacTimeByFrame = dataset[idrecord,range2]
3680
3681 return radacTimeByFrame
3682
3683 def readBeamCode(self, idrecord, range1, range2):
3684 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
3685 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
3686 self.beamCodesFromFile = dataset.value
3687
3688 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
3689 #beamcodeByFrame[range2] = dataset[idrecord, range2]
3690 beamcodeByFrame[range1] = dataset[idrecord, range1]
3691 beamcodeByFrame[range2] = dataset[idrecord, range2]
3692
3693 return beamcodeByFrame
3694
3695
3696 def __setDataByFrame(self):
3697 ndata = 2 # porque es complejo
3698 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
3699 return dataByFrame
3700
3701 def __readDataSet(self):
3702 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
3703 return dataset
3704
3705 def __setDataBlock(self,):
3706 real = self.dataByFrame[:,:,0] #asumo que 0 es real
3707 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
3708 datablock = real + imag*1j #armo el complejo
3709 return datablock
3710
3711 def readSamples_version1(self,idrecord):
3712 #estas tres primeras lineas solo se deben ejecutar una vez
3713 if self.flagIsNewFile:
3714 self.idpulse_range1, self.idpulse_range2 = self.radacHeaderObj.getIndexRangeToPulse(0)
3715 self.dataByFrame = self.__setDataByFrame()
3716 self.beamCodeByFrame = self.readBeamCode(idrecord, self.idpulse_range1, self.idpulse_range2)
3717 self.radacTimeByFrame = self.readRadacTime(idrecord, self.idpulse_range1, self.idpulse_range2)
3718 #reading dataset
3719 self.dataset = self.__readDataSet()
3720
3721 if idrecord == 0:
3722
3723 if len(numpy.where(self.dataByFrame!=0.0)[0]) or len(numpy.where(self.dataByFrame!=0.0)[1]) or len(numpy.where(self.dataByFrame!=0.0)[2]):
3724 #falta agregar una condicion para datos discontinuos
3725 #por defecto une los datos del record anterior
3726 self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]
3727 #timepulse
3728 self.radacTimeByFrame[self.idpulse_range2] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range2]
3729 else:
3730 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]
3731
3732 self.radacTimeByFrame[self.idpulse_range1] = self.radacHeaderObj.radacTime[idrecord, self.idpulse_range1]
3733
3734 datablock = self.__setDataBlock()
3735
3736 return datablock
3737
3738 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord - 1,self.idpulse_range1, :, :]
3739 self.dataByFrame[self.idpulse_range2, :, :] = self.dataset[idrecord, self.idpulse_range2, :, :]
3740 datablock = self.__setDataBlock()
3741 self.flagIsNewFile = 0
3742
3743 self.dataByFrame[self.idpulse_range1, :, :] = self.dataset[idrecord, self.idpulse_range1, :, :]
3744
3745
3746 return datablock
3747
3748
3749 def readSamples(self,idrecord):
3750 if self.flagIsNewFile:
3751 self.dataByFrame = self.__setDataByFrame()
3752 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
3753
3754 #reading dataset
3755 self.dataset = self.__readDataSet()
3756
3757 self.flagIsNewFile = 0
3758 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
3759 self.dataByFrame = self.dataset[idrecord, :, :, :]
3760 datablock = self.__setDataBlock()
3761 return datablock
3762
3763
3764 def readDataBlock(self):
3765
3766 #self.datablock = self.readSamples(self.idrecord_count)
3767 self.datablock = self.readSamples(self.idrecord_count)
3768 #print 'record:', self.idrecord_count
3769
3770 self.idrecord_count += 1
3771 self.profileIndex = 0
3772
3773 if self.idrecord_count >= self.radacHeaderObj.nrecords:
3774 self.idrecord_count = 0
3775 self.flagIsNewFile = 1
3776
3777 def readNextBlock(self):
3778
3779 self.readDataBlock()
3780
3781 if self.flagIsNewFile:
3782 self.__setNextFile()
3783 pass
3784
3785 def __hasNotDataInBuffer(self):
3786 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
3787 if self.profileIndex >= self.radacHeaderObj.npulses:
3788 return 1
3789 return 0
3790
3791
3792 def getData(self):
3793
3794 if self.flagNoMoreFiles:
3795 self.dataOut.flagNoData = True
3796 print 'Process finished'
3797 return 0
3798
3799 if self.__hasNotDataInBuffer():
3800 self.readNextBlock()
3801 # if not( self.readNextBlock() ):
3802 # return 0
3803 # self.getFirstHeader()
3804
3805 if self.datablock == None: # setear esta condicion cuando no hayan datos por leers
3806 self.dataOut.flagNoData = True
3807 return 0
3808
3809 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
3810
3811 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
3812
3813 self.dataOut.flagNoData = False
3814
3815 self.profileIndex += 1
3816
3817 return self.dataOut.data
3818
3819
3820 def run(self, **kwargs):
3821 if not(self.isConfig):
3822 self.setup(**kwargs)
3823 self.isConfig = True
3824
3825 self.getData()
General Comments 0
You need to be logged in to leave comments. Login now