##// END OF EJS Templates
schain
RSS
añadido a jroIO_kamisr.py tratamiento de archivos h5f5 corruptos, mostrará advertencia y continurá en lugar de finalizar el procesamiento
joabAM -
r1423:c7df5d958ab1
parent child
Show More
Side by Side Unified
Context file:
r1423:c7df5d958ab1 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,660 +1,665
1 1 ''''
2 2 Created on Set 9, 2015
3 3
4 4 @author: roj-idl71 Karim Kuyeng
5 5
6 6 @update: 2021, Joab Apaza
7 7 '''
8 8
9 9 import os
10 10 import sys
11 11 import glob
12 12 import fnmatch
13 13 import datetime
14 14 import time
15 15 import re
16 16 import h5py
17 17 import numpy
18 18
19 19 try:
20 20 from gevent import sleep
21 21 except:
22 22 from time import sleep
23 23
24 24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
25 25 from schainpy.model.data.jrodata import Voltage
26 26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
27 27 from numpy import imag
28 from schainpy.utils import log
28 29
29 30
30 31 class AMISRReader(ProcessingUnit):
31 32 '''
32 33 classdocs
33 34 '''
34 35
35 36 def __init__(self):
36 37 '''
37 38 Constructor
38 39 '''
39 40
40 41 ProcessingUnit.__init__(self)
41 42
42 43 self.set = None
43 44 self.subset = None
44 45 self.extension_file = '.h5'
45 46 self.dtc_str = 'dtc'
46 47 self.dtc_id = 0
47 48 self.status = True
48 49 self.isConfig = False
49 50 self.dirnameList = []
50 51 self.filenameList = []
51 52 self.fileIndex = None
52 53 self.flagNoMoreFiles = False
53 54 self.flagIsNewFile = 0
54 55 self.filename = ''
55 56 self.amisrFilePointer = None
56 57 self.realBeamCode = []
57 58 self.beamCodeMap = None
58 59 self.azimuthList = []
59 60 self.elevationList = []
60 61 self.dataShape = None
61 62 self.flag_old_beams = False
62 63
63 64
64 65 self.profileIndex = 0
65 66
66 67
67 68 self.beamCodeByFrame = None
68 69 self.radacTimeByFrame = None
69 70
70 71 self.dataset = None
71 72
72 73 self.__firstFile = True
73 74
74 75 self.buffer = None
75 76
76 77 self.timezone = 'ut'
77 78
78 79 self.__waitForNewFile = 20
79 80 self.__filename_online = None
80 81 #Is really necessary create the output object in the initializer
81 82 self.dataOut = Voltage()
82 83 self.dataOut.error=False
83 84
84 85
85 86 def setup(self,path=None,
86 87 startDate=None,
87 88 endDate=None,
88 89 startTime=None,
89 90 endTime=None,
90 91 walk=True,
91 92 timezone='ut',
92 93 all=0,
93 94 code = None,
94 95 nCode = 0,
95 96 nBaud = 0,
96 97 online=False,
97 98 old_beams=False):
98 99
99 100
100 101
101 102 self.timezone = timezone
102 103 self.all = all
103 104 self.online = online
104 105 self.flag_old_beams = old_beams
105 106 self.code = code
106 107 self.nCode = int(nCode)
107 108 self.nBaud = int(nBaud)
108 109
109 110
110 111
111 112 #self.findFiles()
112 113 if not(online):
113 114 #Busqueda de archivos offline
114 115 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
115 116 else:
116 117 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
117 118
118 119 if not(self.filenameList):
119 120 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
120 121 sys.exit()
121 122
122 123 self.fileIndex = 0
123 124
124 125 self.readNextFile(online)
125 126
126 127 '''
127 128 Add code
128 129 '''
129 130 self.isConfig = True
130 131 # print("Setup Done")
131 132 pass
132 133
133 134
134 135 def readAMISRHeader(self,fp):
135 136
136 137 if self.isConfig and (not self.flagNoMoreFiles):
137 138 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
138 139 if self.dataShape != newShape and newShape != None:
139 140 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
140 141 print(self.dataShape,newShape,"\n")
141 142 return 0
142 143 else:
143 144 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
144 145
145 146
146 147 header = 'Raw11/Data/RadacHeader'
147 148 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
148 149 if (self.startDate> datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambió la forma de extracción de Apuntes el 17 o forzar con flag de reorganización
149 150 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
150 151 self.trueBeams = self.beamcodeFile.split("\n")
151 152 self.trueBeams.pop()#remove last
152 153 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
153 154 self.beamCode = [int(x, 16) for x in self.realBeamCode]
154 155 else:
155 156 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
156 157 self.beamCode = _beamCode[0,:]
157 158
158 159 if self.beamCodeMap == None:
159 160 self.beamCodeMap = fp['Setup/BeamcodeMap']
160 161 for beam in self.beamCode:
161 162 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
162 163 beamAziElev = beamAziElev[0].squeeze()
163 164 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
164 165 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
165 166 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
166 167 #print(self.beamCode)
167 168 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
168 169 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
169 170 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
170 171 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
171 172 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
172 173 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
173 174 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
174 175 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
175 176 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
176 177 self.frequency = fp.get('Rx/Frequency')
177 178 txAus = fp.get('Raw11/Data/Pulsewidth')
178 179
179 180
180 181 self.nblocks = self.pulseCount.shape[0] #nblocks
181 182
182 183 self.nprofiles = self.pulseCount.shape[1] #nprofile
183 184 self.nsa = self.nsamplesPulse[0,0] #ngates
184 185 self.nchannels = len(self.beamCode)
185 186 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
186 187 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
187 188 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
188 189
189 190 #filling radar controller header parameters
190 191 self.__ippKm = self.ippSeconds *.15*1e6 # in km
191 192 self.__txA = (txAus[()])*.15 #(ipp[us]*.15km/1us) in km
192 193 self.__txB = 0
193 194 nWindows=1
194 195 self.__nSamples = self.nsa
195 196 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
196 197 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
197 198
198 199 #for now until understand why the code saved is different (code included even though code not in tuf file)
199 200 #self.__codeType = 0
200 201 # self.__nCode = None
201 202 # self.__nBaud = None
202 203 self.__code = self.code
203 204 self.__codeType = 0
204 205 if self.code != None:
205 206 self.__codeType = 1
206 207 self.__nCode = self.nCode
207 208 self.__nBaud = self.nBaud
208 209 #self.__code = 0
209 210
210 211 #filling system header parameters
211 212 self.__nSamples = self.nsa
212 213 self.newProfiles = self.nprofiles/self.nchannels
213 214 self.__channelList = list(range(self.nchannels))
214 215
215 216 self.__frequency = self.frequency[0][0]
216 217
217 218
218 219 return 1
219 220
220 221
221 222 def createBuffers(self):
222 223
223 224 pass
224 225
225 226 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
226 227 self.path = path
227 228 self.startDate = startDate
228 229 self.endDate = endDate
229 230 self.startTime = startTime
230 231 self.endTime = endTime
231 232 self.walk = walk
232 233
233 234 def __checkPath(self):
234 235 if os.path.exists(self.path):
235 236 self.status = 1
236 237 else:
237 238 self.status = 0
238 239 print('Path:%s does not exists'%self.path)
239 240
240 241 return
241 242
242 243
243 244 def __selDates(self, amisr_dirname_format):
244 245 try:
245 246 year = int(amisr_dirname_format[0:4])
246 247 month = int(amisr_dirname_format[4:6])
247 248 dom = int(amisr_dirname_format[6:8])
248 249 thisDate = datetime.date(year,month,dom)
249 250 #margen de un día extra, igual luego se filtra for fecha y hora
250 251 if (thisDate>=(self.startDate - datetime.timedelta(days=1)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
251 252 return amisr_dirname_format
252 253 except:
253 254 return None
254 255
255 256
256 257 def __findDataForDates(self,online=False):
257 258
258 259 if not(self.status):
259 260 return None
260 261
261 262 pat = '\d+.\d+'
262 263 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
263 264 dirnameList = [x for x in dirnameList if x!=None]
264 265 dirnameList = [x.string for x in dirnameList]
265 266 if not(online):
266 267 dirnameList = [self.__selDates(x) for x in dirnameList]
267 268 dirnameList = [x for x in dirnameList if x!=None]
268 269 if len(dirnameList)>0:
269 270 self.status = 1
270 271 self.dirnameList = dirnameList
271 272 self.dirnameList.sort()
272 273 else:
273 274 self.status = 0
274 275 return None
275 276
276 277 def __getTimeFromData(self):
277 278 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
278 279 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
279 280
280 281 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
281 282 print('........................................')
282 283 filter_filenameList = []
283 284 self.filenameList.sort()
285 total_files = len(self.filenameList)
284 286 #for i in range(len(self.filenameList)-1):
285 for i in range(len(self.filenameList)):
287 for i in range(total_files):
286 288 filename = self.filenameList[i]
287 fp = h5py.File(filename,'r')
288 time_str = fp.get('Time/RadacTimeString')
289
290 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
291 #startDateTimeStr_File = "2019-12-16 09:21:11"
292 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
293 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
294
295 #endDateTimeStr_File = "2019-12-16 11:10:11"
296 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
297 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
298 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
299
300 fp.close()
301
302 #print("check time", startDateTime_File)
303 if self.timezone == 'lt':
304 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
305 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
306 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
307 filter_filenameList.append(filename)
308
309 if (startDateTime_File>endDateTime_Reader):
310 break
311
289 #print("file-> ",filename)
290 try:
291 fp = h5py.File(filename,'r')
292 time_str = fp.get('Time/RadacTimeString')
293
294 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
295 #startDateTimeStr_File = "2019-12-16 09:21:11"
296 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
297 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
298
299 #endDateTimeStr_File = "2019-12-16 11:10:11"
300 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
301 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
302 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
303
304 fp.close()
305
306 #print("check time", startDateTime_File)
307 if self.timezone == 'lt':
308 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
309 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
310 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
311 filter_filenameList.append(filename)
312
313 if (startDateTime_File>endDateTime_Reader):
314 break
315 except Exception as e:
316 log.warning("Error opening file {} -> {}".format(os.path.split(filename)[1],e))
312 317
313 318 filter_filenameList.sort()
314 319 self.filenameList = filter_filenameList
315 320
316 321 return 1
317 322
318 323 def __filterByGlob1(self, dirName):
319 324 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
320 325 filter_files.sort()
321 326 filterDict = {}
322 327 filterDict.setdefault(dirName)
323 328 filterDict[dirName] = filter_files
324 329 return filterDict
325 330
326 331 def __getFilenameList(self, fileListInKeys, dirList):
327 332 for value in fileListInKeys:
328 333 dirName = list(value.keys())[0]
329 334 for file in value[dirName]:
330 335 filename = os.path.join(dirName, file)
331 336 self.filenameList.append(filename)
332 337
333 338
334 339 def __selectDataForTimes(self, online=False):
335 340 #aun no esta implementado el filtro for tiempo
336 341 if not(self.status):
337 342 return None
338 343
339 344 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
340 345 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
341 346 self.__getFilenameList(fileListInKeys, dirList)
342 347 if not(online):
343 348 #filtro por tiempo
344 349 if not(self.all):
345 350 self.__getTimeFromData()
346 351
347 352 if len(self.filenameList)>0:
348 353 self.status = 1
349 354 self.filenameList.sort()
350 355 else:
351 356 self.status = 0
352 357 return None
353 358
354 359 else:
355 360 #get the last file - 1
356 361 self.filenameList = [self.filenameList[-2]]
357 362 new_dirnameList = []
358 363 for dirname in self.dirnameList:
359 364 junk = numpy.array([dirname in x for x in self.filenameList])
360 365 junk_sum = junk.sum()
361 366 if junk_sum > 0:
362 367 new_dirnameList.append(dirname)
363 368 self.dirnameList = new_dirnameList
364 369 return 1
365 370
366 371 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
367 372 endTime=datetime.time(23,59,59),walk=True):
368 373
369 374 if endDate ==None:
370 375 startDate = datetime.datetime.utcnow().date()
371 376 endDate = datetime.datetime.utcnow().date()
372 377
373 378 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
374 379
375 380 self.__checkPath()
376 381
377 382 self.__findDataForDates(online=True)
378 383
379 384 self.dirnameList = [self.dirnameList[-1]]
380 385
381 386 self.__selectDataForTimes(online=True)
382 387
383 388 return
384 389
385 390
386 391 def searchFilesOffLine(self,
387 392 path,
388 393 startDate,
389 394 endDate,
390 395 startTime=datetime.time(0,0,0),
391 396 endTime=datetime.time(23,59,59),
392 397 walk=True):
393 398
394 399 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
395 400
396 401 self.__checkPath()
397 402
398 403 self.__findDataForDates()
399 404
400 405 self.__selectDataForTimes()
401 406
402 407 for i in range(len(self.filenameList)):
403 408 print("%s" %(self.filenameList[i]))
404 409
405 410 return
406 411
407 412 def __setNextFileOffline(self):
408 413
409 414 try:
410 415 self.filename = self.filenameList[self.fileIndex]
411 416 self.amisrFilePointer = h5py.File(self.filename,'r')
412 417 self.fileIndex += 1
413 418 except:
414 419 self.flagNoMoreFiles = 1
415 420 raise schainpy.admin.SchainError('No more files to read')
416 421 return 0
417 422
418 423 self.flagIsNewFile = 1
419 424 print("Setting the file: %s"%self.filename)
420 425
421 426 return 1
422 427
423 428
424 429 def __setNextFileOnline(self):
425 430 filename = self.filenameList[0]
426 431 if self.__filename_online != None:
427 432 self.__selectDataForTimes(online=True)
428 433 filename = self.filenameList[0]
429 434 wait = 0
430 435 self.__waitForNewFile=300 ## DEBUG:
431 436 while self.__filename_online == filename:
432 437 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
433 438 if wait == 5:
434 439 self.flagNoMoreFiles = 1
435 440 return 0
436 441 sleep(self.__waitForNewFile)
437 442 self.__selectDataForTimes(online=True)
438 443 filename = self.filenameList[0]
439 444 wait += 1
440 445
441 446 self.__filename_online = filename
442 447
443 448 self.amisrFilePointer = h5py.File(filename,'r')
444 449 self.flagIsNewFile = 1
445 450 self.filename = filename
446 451 print("Setting the file: %s"%self.filename)
447 452 return 1
448 453
449 454
450 455 def readData(self):
451 456 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
452 457 re = buffer[:,:,:,0]
453 458 im = buffer[:,:,:,1]
454 459 dataset = re + im*1j
455 460
456 461 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
457 462 timeset = self.radacTime[:,0]
458 463
459 464 return dataset,timeset
460 465
461 466 def reshapeData(self):
462 467 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
463 468 channels = self.beamCodeByPulse[0,:]
464 469 nchan = self.nchannels
465 470 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
466 471 nblocks = self.nblocks
467 472 nsamples = self.nsa
468 473
469 474 #Dimensions : nChannels, nProfiles, nSamples
470 475 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
471 476 ############################################
472 477
473 478 for thisChannel in range(nchan):
474 479 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
475 480
476 481
477 482 new_block = numpy.transpose(new_block, (1,0,2,3))
478 483 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
479 484
480 485 return new_block
481 486
482 487 def updateIndexes(self):
483 488
484 489 pass
485 490
486 491 def fillJROHeader(self):
487 492
488 493 #fill radar controller header
489 494 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
490 495 txA=self.__txA,
491 496 txB=0,
492 497 nWindows=1,
493 498 nHeights=self.__nSamples,
494 499 firstHeight=self.__firstHeight,
495 500 deltaHeight=self.__deltaHeight,
496 501 codeType=self.__codeType,
497 502 nCode=self.__nCode, nBaud=self.__nBaud,
498 503 code = self.__code,
499 504 fClock=1)
500 505
501 506 #fill system header
502 507 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
503 508 nProfiles=self.newProfiles,
504 509 nChannels=len(self.__channelList),
505 510 adcResolution=14,
506 511 pciDioBusWidth=32)
507 512
508 513 self.dataOut.type = "Voltage"
509 514 self.dataOut.data = None
510 515 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
511 516 # self.dataOut.nChannels = 0
512 517
513 518 # self.dataOut.nHeights = 0
514 519
515 520 self.dataOut.nProfiles = self.newProfiles*self.nblocks
516 521 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
517 522 ranges = numpy.reshape(self.rangeFromFile[()],(-1))
518 523 self.dataOut.heightList = ranges/1000.0 #km
519 524 self.dataOut.channelList = self.__channelList
520 525 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
521 526
522 527 # self.dataOut.channelIndexList = None
523 528
524 529
525 530 self.dataOut.azimuthList = numpy.array(self.azimuthList)
526 531 self.dataOut.elevationList = numpy.array(self.elevationList)
527 532 self.dataOut.codeList = numpy.array(self.beamCode)
528 533 #print(self.dataOut.elevationList)
529 534 self.dataOut.flagNoData = True
530 535
531 536 #Set to TRUE if the data is discontinuous
532 537 self.dataOut.flagDiscontinuousBlock = False
533 538
534 539 self.dataOut.utctime = None
535 540
536 541 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
537 542 if self.timezone == 'lt':
538 543 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
539 544 else:
540 545 self.dataOut.timeZone = 0 #by default time is UTC
541 546
542 547 self.dataOut.dstFlag = 0
543 548 self.dataOut.errorCount = 0
544 549 self.dataOut.nCohInt = 1
545 550 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
546 551 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
547 552 self.dataOut.flagShiftFFT = False
548 553 self.dataOut.ippSeconds = self.ippSeconds
549 554
550 555 #Time interval between profiles
551 556 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
552 557
553 558 self.dataOut.frequency = self.__frequency
554 559 self.dataOut.realtime = self.online
555 560 pass
556 561
557 562 def readNextFile(self,online=False):
558 563
559 564 if not(online):
560 565 newFile = self.__setNextFileOffline()
561 566 else:
562 567 newFile = self.__setNextFileOnline()
563 568
564 569 if not(newFile):
565 570 self.dataOut.error = True
566 571 return 0
567 572
568 573 if not self.readAMISRHeader(self.amisrFilePointer):
569 574 self.dataOut.error = True
570 575 return 0
571 576
572 577 self.createBuffers()
573 578 self.fillJROHeader()
574 579
575 580 #self.__firstFile = False
576 581
577 582
578 583
579 584 self.dataset,self.timeset = self.readData()
580 585
581 586 if self.endDate!=None:
582 587 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
583 588 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
584 589 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
585 590 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
586 591 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
587 592 if self.timezone == 'lt':
588 593 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
589 594 if (startDateTime_File>endDateTime_Reader):
590 595 return 0
591 596
592 597 self.jrodataset = self.reshapeData()
593 598 #----self.updateIndexes()
594 599 self.profileIndex = 0
595 600
596 601 return 1
597 602
598 603
599 604 def __hasNotDataInBuffer(self):
600 605 if self.profileIndex >= (self.newProfiles*self.nblocks):
601 606 return 1
602 607 return 0
603 608
604 609
605 610 def getData(self):
606 611
607 612 if self.flagNoMoreFiles:
608 613 self.dataOut.flagNoData = True
609 614 return 0
610 615
611 616 if self.__hasNotDataInBuffer():
612 617 if not (self.readNextFile(self.online)):
613 618 return 0
614 619
615 620
616 621 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
617 622 self.dataOut.flagNoData = True
618 623 return 0
619 624
620 625 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
621 626
622 627 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
623 628
624 629 #print("R_t",self.timeset)
625 630
626 631 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
627 632 #verificar basic header de jro data y ver si es compatible con este valor
628 633 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
629 634 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
630 635 indexblock = self.profileIndex/self.newProfiles
631 636 #print (indexblock, indexprof)
632 637 diffUTC = 0
633 638 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
634 639
635 640 #print("utc :",indexblock," __ ",t_comp)
636 641 #print(numpy.shape(self.timeset))
637 642 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
638 643 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
639 644
640 645 self.dataOut.profileIndex = self.profileIndex
641 646 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
642 647 self.dataOut.flagNoData = False
643 648 # if indexprof == 0:
644 649 # print("kamisr: ",self.dataOut.utctime)
645 650
646 651 self.profileIndex += 1
647 652
648 653 return self.dataOut.data #retorno necesario??
649 654
650 655
651 656 def run(self, **kwargs):
652 657 '''
653 658 This method will be called many times so here you should put all your code
654 659 '''
655 660 #print("running kamisr")
656 661 if not self.isConfig:
657 662 self.setup(**kwargs)
658 663 self.isConfig = True
659 664
660 665 self.getData()
General Comments 0
You need to be logged in to leave comments. Login now