##// END OF EJS Templates
schain-jc
RSS
To be able to use this jroIO_kamisr.py change on the io/__init__.py the jroIO_amisr to jro_IOkamisr...
Karim Valdez -
r665:6bd7c358b003
parent child
Show More
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (675 lines changed) Show them Hide them
@@ -0,0 +1,675
1 '''
2 Created on Set 9, 2015
3
4 @author: roj-idl71 Karim Kuyeng
5 '''
6
7 import os
8 import sys
9 import glob
10 import fnmatch
11 import datetime
12 import time
13 import re
14 import h5py
15 import numpy
16
17 try:
18 from gevent import sleep
19 except:
20 from time import sleep
21
22 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
23 from schainpy.model.data.jrodata import Voltage
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
25 from numpy import imag
26
27 class AMISRReader(ProcessingUnit):
28 '''
29 classdocs
30 '''
31
32 def __init__(self):
33 '''
34 Constructor
35 '''
36
37 ProcessingUnit.__init__(self)
38
39 self.set = None
40 self.subset = None
41 self.extension_file = '.h5'
42 self.dtc_str = 'dtc'
43 self.dtc_id = 0
44 self.status = True
45 self.isConfig = False
46 self.dirnameList = []
47 self.filenameList = []
48 self.fileIndex = None
49 self.flagNoMoreFiles = False
50 self.flagIsNewFile = 0
51 self.filename = ''
52 self.amisrFilePointer = None
53
54
55 self.dataset = None
56
57
58
59
60 self.profileIndex = 0
61
62
63 self.beamCodeByFrame = None
64 self.radacTimeByFrame = None
65
66 self.dataset = None
67
68
69
70
71 self.__firstFile = True
72
73 self.buffer = None
74
75
76 self.timezone = 'ut'
77
78 self.__waitForNewFile = 20
79 self.__filename_online = None
80 #Is really necessary create the output object in the initializer
81 self.dataOut = Voltage()
82
83 def setup(self,path=None,
84 startDate=None,
85 endDate=None,
86 startTime=None,
87 endTime=None,
88 walk=True,
89 timezone='ut',
90 all=0,
91 code = None,
92 nCode = 0,
93 nBaud = 0,
94 online=False):
95
96 self.timezone = timezone
97 self.all = all
98 self.online = online
99
100 self.code = code
101 self.nCode = int(nCode)
102 self.nBaud = int(nBaud)
103
104
105
106 #self.findFiles()
107 if not(online):
108 #Busqueda de archivos offline
109 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
110 else:
111 self.__searchFilesOnline(path, startDate, endDate, startTime,endTime,walk)
112
113 if not(self.filenameList):
114 print "There is no files into the folder: %s"%(path)
115
116 sys.exit(-1)
117
118 self.fileIndex = -1
119
120 self.readNextFile(online)
121
122 '''
123 Add code
124 '''
125 self.isConfig = True
126
127 pass
128
129
130 def readAMISRHeader(self,fp):
131 header = 'Raw11/Data/RadacHeader'
132 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
133 self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
134 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
135 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
136 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
137 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
138 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
139 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
140 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
141 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
142 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
143 self.frequency = fp.get('Rx/Frequency')
144 txAus = fp.get('Raw11/Data/Pulsewidth')
145
146
147 self.nblocks = self.pulseCount.shape[0] #nblocks
148
149 self.nprofiles = self.pulseCount.shape[1] #nprofile
150 self.nsa = self.nsamplesPulse[0,0] #ngates
151 self.nchannels = self.beamCode.shape[1]
152 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
153 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
154 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
155
156 #filling radar controller header parameters
157 self.__ippKm = self.ippSeconds *.15*1e6 # in km
158 self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
159 self.__txB = 0
160 nWindows=1
161 self.__nSamples = self.nsa
162 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
163 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
164
165 #for now until understand why the code saved is different (code included even though code not in tuf file)
166 #self.__codeType = 0
167 # self.__nCode = None
168 # self.__nBaud = None
169 self.__code = self.code
170 self.__codeType = 0
171 if self.code != None:
172 self.__codeType = 1
173 self.__nCode = self.nCode
174 self.__nBaud = self.nBaud
175 #self.__code = 0
176
177 #filling system header parameters
178 self.__nSamples = self.nsa
179 self.newProfiles = self.nprofiles/self.nchannels
180 self.__channelList = range(self.nchannels)
181
182 self.__frequency = self.frequency[0][0]
183
184
185
186 def createBuffers(self):
187
188 pass
189
190 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
191 self.path = path
192 self.startDate = startDate
193 self.endDate = endDate
194 self.startTime = startTime
195 self.endTime = endTime
196 self.walk = walk
197
198 def __checkPath(self):
199 if os.path.exists(self.path):
200 self.status = 1
201 else:
202 self.status = 0
203 print 'Path:%s does not exists'%self.path
204
205 return
206
207
208 def __selDates(self, amisr_dirname_format):
209 try:
210 year = int(amisr_dirname_format[0:4])
211 month = int(amisr_dirname_format[4:6])
212 dom = int(amisr_dirname_format[6:8])
213 thisDate = datetime.date(year,month,dom)
214
215 if (thisDate>=self.startDate and thisDate <= self.endDate):
216 return amisr_dirname_format
217 except:
218 return None
219
220
221 def __findDataForDates(self,online=False):
222
223 if not(self.status):
224 return None
225
226 pat = '\d+.\d+'
227 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
228 dirnameList = filter(lambda x:x!=None,dirnameList)
229 dirnameList = [x.string for x in dirnameList]
230 if not(online):
231 dirnameList = [self.__selDates(x) for x in dirnameList]
232 dirnameList = filter(lambda x:x!=None,dirnameList)
233 if len(dirnameList)>0:
234 self.status = 1
235 self.dirnameList = dirnameList
236 self.dirnameList.sort()
237 else:
238 self.status = 0
239 return None
240
241 def __getTimeFromData(self):
242 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
243 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
244
245 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
246 print '........................................'
247 filter_filenameList = []
248 self.filenameList.sort()
249 #for i in range(len(self.filenameList)-1):
250 for i in range(len(self.filenameList)):
251 filename = self.filenameList[i]
252 fp = h5py.File(filename,'r')
253 time_str = fp.get('Time/RadacTimeString')
254
255 startDateTimeStr_File = time_str[0][0].split('.')[0]
256 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
257 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
258
259 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
260 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
261 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
262
263 fp.close()
264
265 if self.timezone == 'lt':
266 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
267 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
268
269 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
270 #self.filenameList.remove(filename)
271 filter_filenameList.append(filename)
272
273 if (endDateTime_File>=endDateTime_Reader):
274 break
275
276
277 filter_filenameList.sort()
278 self.filenameList = filter_filenameList
279 return 1
280
281 def __filterByGlob1(self, dirName):
282 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
283 filter_files.sort()
284 filterDict = {}
285 filterDict.setdefault(dirName)
286 filterDict[dirName] = filter_files
287 return filterDict
288
289 def __getFilenameList(self, fileListInKeys, dirList):
290 for value in fileListInKeys:
291 dirName = value.keys()[0]
292 for file in value[dirName]:
293 filename = os.path.join(dirName, file)
294 self.filenameList.append(filename)
295
296
297 def __selectDataForTimes(self, online=False):
298 #aun no esta implementado el filtro for tiempo
299 if not(self.status):
300 return None
301
302 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
303
304 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
305
306 self.__getFilenameList(fileListInKeys, dirList)
307 if not(online):
308 #filtro por tiempo
309 if not(self.all):
310 self.__getTimeFromData()
311
312 if len(self.filenameList)>0:
313 self.status = 1
314 self.filenameList.sort()
315 else:
316 self.status = 0
317 return None
318
319 else:
320 #get the last file - 1
321 self.filenameList = [self.filenameList[-2]]
322
323 new_dirnameList = []
324 for dirname in self.dirnameList:
325 junk = numpy.array([dirname in x for x in self.filenameList])
326 junk_sum = junk.sum()
327 if junk_sum > 0:
328 new_dirnameList.append(dirname)
329 self.dirnameList = new_dirnameList
330 return 1
331
332 def __searchFilesOnline(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 endTime=datetime.time(23,59,59),walk=True):
334
335 if endDate ==None:
336 startDate = datetime.datetime.utcnow().date()
337 endDate = datetime.datetime.utcnow().date()
338
339 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
340
341 self.__checkPath()
342
343 self.__findDataForDates(online=True)
344
345 self.dirnameList = [self.dirnameList[-1]]
346
347 self.__selectDataForTimes(online=True)
348
349 return
350
351
352 def __searchFilesOffline(self,
353 path,
354 startDate,
355 endDate,
356 startTime=datetime.time(0,0,0),
357 endTime=datetime.time(23,59,59),
358 walk=True):
359
360 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
361
362 self.__checkPath()
363
364 self.__findDataForDates()
365
366 self.__selectDataForTimes()
367
368 for i in range(len(self.filenameList)):
369 print "%s" %(self.filenameList[i])
370
371 return
372
373 def __setNextFileOffline(self):
374 idFile = self.fileIndex
375
376 while (True):
377 idFile += 1
378 if not(idFile < len(self.filenameList)):
379 self.flagNoMoreFiles = 1
380 print "No more Files"
381 return 0
382
383 filename = self.filenameList[idFile]
384
385 amisrFilePointer = h5py.File(filename,'r')
386
387 break
388
389 self.flagIsNewFile = 1
390 self.fileIndex = idFile
391 self.filename = filename
392
393 self.amisrFilePointer = amisrFilePointer
394
395 print "Setting the file: %s"%self.filename
396
397 return 1
398
399
400 def __setNextFileOnline(self):
401 filename = self.filenameList[0]
402 if self.__filename_online != None:
403 self.__selectDataForTimes(online=True)
404 filename = self.filenameList[0]
405 wait = 0
406 while self.__filename_online == filename:
407 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
408 if wait == 5:
409 return 0
410 sleep(self.__waitForNewFile)
411 self.__selectDataForTimes(online=True)
412 filename = self.filenameList[0]
413 wait += 1
414
415 self.__filename_online = filename
416
417 self.amisrFilePointer = h5py.File(filename,'r')
418 self.flagIsNewFile = 1
419 self.filename = filename
420 print "Setting the file: %s"%self.filename
421 return 1
422
423
424 def readData(self):
425 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
426 re = buffer[:,:,:,0]
427 im = buffer[:,:,:,1]
428 dataset = re + im*1j
429 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
430 timeset = self.radacTime[:,0]
431 return dataset,timeset
432
433 def reshapeData(self):
434 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
435 channels = self.beamCodeByPulse[0,:]
436 nchan = self.nchannels
437 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
438 nblocks = self.nblocks
439 nsamples = self.nsa
440
441 #Dimensions : nChannels, nProfiles, nSamples
442 new_block = numpy.empty((nblocks, nchan, self.newProfiles, nsamples), dtype="complex64")
443 ############################################
444
445 for thisChannel in range(nchan):
446 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[0][thisChannel])[0],:]
447
448
449 new_block = numpy.transpose(new_block, (1,0,2,3))
450 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
451
452 return new_block
453
454 def updateIndexes(self):
455
456 pass
457
458 def fillJROHeader(self):
459
460 #fill radar controller header
461 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
462 txA=self.__txA,
463 txB=0,
464 nWindows=1,
465 nHeights=self.__nSamples,
466 firstHeight=self.__firstHeight,
467 deltaHeight=self.__deltaHeight,
468 codeType=self.__codeType,
469 nCode=self.__nCode, nBaud=self.__nBaud,
470 code = self.__code,
471 fClock=1)
472
473
474
475 #fill system header
476 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
477 nProfiles=self.newProfiles,
478 nChannels=len(self.__channelList),
479 adcResolution=14,
480 pciDioBusWith=32)
481
482 self.dataOut.type = "Voltage"
483
484 self.dataOut.data = None
485
486 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
487
488 # self.dataOut.nChannels = 0
489
490 # self.dataOut.nHeights = 0
491
492 self.dataOut.nProfiles = self.newProfiles*self.nblocks
493
494 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
495 ranges = numpy.reshape(self.rangeFromFile.value,(-1))
496 self.dataOut.heightList = ranges/1000.0 #km
497
498
499 self.dataOut.channelList = self.__channelList
500
501 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
502
503 # self.dataOut.channelIndexList = None
504
505 self.dataOut.flagNoData = True
506
507 #Set to TRUE if the data is discontinuous
508 self.dataOut.flagDiscontinuousBlock = False
509
510 self.dataOut.utctime = None
511
512 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
513 if self.timezone == 'lt':
514 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
515 else:
516 self.dataOut.timeZone = 0 #by default time is UTC
517
518 self.dataOut.dstFlag = 0
519
520 self.dataOut.errorCount = 0
521
522 self.dataOut.nCohInt = 1
523
524 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
525
526 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
527
528 self.dataOut.flagShiftFFT = False
529
530 self.dataOut.ippSeconds = self.ippSeconds
531
532 #Time interval between profiles
533 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
534
535 self.dataOut.frequency = self.__frequency
536
537 self.dataOut.realtime = self.online
538 pass
539
540 def readNextFile(self,online=False):
541
542 if not(online):
543 newFile = self.__setNextFileOffline()
544 else:
545 newFile = self.__setNextFileOnline()
546
547 if not(newFile):
548 return 0
549
550 #if self.__firstFile:
551 self.readAMISRHeader(self.amisrFilePointer)
552 self.createBuffers()
553 self.fillJROHeader()
554 #self.__firstFile = False
555
556
557
558 self.dataset,self.timeset = self.readData()
559
560 if self.endDate!=None:
561 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
562 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
563 startDateTimeStr_File = time_str[0][0].split('.')[0]
564 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
565 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
566 if self.timezone == 'lt':
567 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
568 if (startDateTime_File>endDateTime_Reader):
569 return 0
570
571 self.jrodataset = self.reshapeData()
572 #----self.updateIndexes()
573 self.profileIndex = 0
574
575 return 1
576
577
578 def __hasNotDataInBuffer(self):
579 if self.profileIndex >= (self.newProfiles*self.nblocks):
580 return 1
581 return 0
582
583
584 def getData(self):
585
586 if self.flagNoMoreFiles:
587 self.dataOut.flagNoData = True
588 print 'Process finished'
589 return 0
590
591 if self.__hasNotDataInBuffer():
592 if not (self.readNextFile(self.online)):
593 return 0
594
595
596 if self.dataset is None: # setear esta condicion cuando no hayan datos por leers
597 self.dataOut.flagNoData = True
598 return 0
599
600 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
601
602 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
603
604 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
605 #verificar basic header de jro data y ver si es compatible con este valor
606 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
607 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
608 indexblock = self.profileIndex/self.newProfiles
609 #print indexblock, indexprof
610 self.dataOut.utctime = self.timeset[indexblock] + (indexprof * self.ippSeconds * self.nchannels)
611 self.dataOut.profileIndex = self.profileIndex
612 self.dataOut.flagNoData = False
613 # if indexprof == 0:
614 # print self.dataOut.utctime
615
616 self.profileIndex += 1
617
618 return self.dataOut.data
619
620
621 def run(self, **kwargs):
622 '''
623 This method will be called many times so here you should put all your code
624 '''
625
626 if not self.isConfig:
627 self.setup(**kwargs)
628 self.isConfig = True
629
630 self.getData()
631
632 class Writer(Operation):
633 '''
634 classdocs
635 '''
636
637 def __init__(self):
638 '''
639 Constructor
640 '''
641 self.dataOut = None
642
643 self.isConfig = False
644
645 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
646 '''
647 In this method we should set all initial parameters.
648
649 Input:
650 dataIn : Input data will also be outputa data
651
652 '''
653 self.dataOut = dataIn
654
655
656
657
658
659 self.isConfig = True
660
661 return
662
663 def run(self, dataIn, **kwargs):
664 '''
665 This method will be called many times so here you should put all your code
666
667 Inputs:
668
669 dataIn : object with the data
670
671 '''
672
673 if not self.isConfig:
674 self.setup(dataIn, **kwargs)
675 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now