|
|
@@
-1,660
+1,662
|
|
|
1
|
'''
|
|
1
|
'''
|
|
|
2
|
Created on Set 9, 2015
|
|
2
|
Created on Set 9, 2015
|
|
|
3
|
|
|
3
|
|
|
|
4
|
@author: roj-idl71 Karim Kuyeng
|
|
4
|
@author: roj-idl71 Karim Kuyeng
|
|
|
|
|
|
5
|
|
|
|
|
|
|
6
|
@update: 2021, Joab Apaza
|
|
|
5
|
'''
|
|
7
|
'''
|
|
|
6
|
|
|
8
|
|
|
|
7
|
import os
|
|
9
|
import os
|
|
|
8
|
import sys
|
|
10
|
import sys
|
|
|
9
|
import glob
|
|
11
|
import glob
|
|
|
10
|
import fnmatch
|
|
12
|
import fnmatch
|
|
|
11
|
import datetime
|
|
13
|
import datetime
|
|
|
12
|
import time
|
|
14
|
import time
|
|
|
13
|
import re
|
|
15
|
import re
|
|
|
14
|
import h5py
|
|
16
|
import h5py
|
|
|
15
|
import numpy
|
|
17
|
import numpy
|
|
|
16
|
|
|
18
|
|
|
|
17
|
try:
|
|
19
|
try:
|
|
|
18
|
from gevent import sleep
|
|
20
|
from gevent import sleep
|
|
|
19
|
except:
|
|
21
|
except:
|
|
|
20
|
from time import sleep
|
|
22
|
from time import sleep
|
|
|
21
|
|
|
23
|
|
|
|
22
|
from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
|
|
24
|
from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
|
|
|
23
|
from schainpy.model.data.jrodata import Voltage
|
|
25
|
from schainpy.model.data.jrodata import Voltage
|
|
|
24
|
from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
|
|
26
|
from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
|
|
|
25
|
from numpy import imag
|
|
27
|
from numpy import imag
|
|
|
26
|
|
|
28
|
|
|
|
27
|
|
|
29
|
|
|
|
28
|
class AMISRReader(ProcessingUnit):
|
|
30
|
class AMISRReader(ProcessingUnit):
|
|
|
29
|
'''
|
|
31
|
'''
|
|
|
30
|
classdocs
|
|
32
|
classdocs
|
|
|
31
|
'''
|
|
33
|
'''
|
|
|
32
|
|
|
34
|
|
|
|
33
|
def __init__(self):
|
|
35
|
def __init__(self):
|
|
|
34
|
'''
|
|
36
|
'''
|
|
|
35
|
Constructor
|
|
37
|
Constructor
|
|
|
36
|
'''
|
|
38
|
'''
|
|
|
37
|
|
|
39
|
|
|
|
38
|
ProcessingUnit.__init__(self)
|
|
40
|
ProcessingUnit.__init__(self)
|
|
|
39
|
|
|
41
|
|
|
|
40
|
self.set = None
|
|
42
|
self.set = None
|
|
|
41
|
self.subset = None
|
|
43
|
self.subset = None
|
|
|
42
|
self.extension_file = '.h5'
|
|
44
|
self.extension_file = '.h5'
|
|
|
43
|
self.dtc_str = 'dtc'
|
|
45
|
self.dtc_str = 'dtc'
|
|
|
44
|
self.dtc_id = 0
|
|
46
|
self.dtc_id = 0
|
|
|
45
|
self.status = True
|
|
47
|
self.status = True
|
|
|
46
|
self.isConfig = False
|
|
48
|
self.isConfig = False
|
|
|
47
|
self.dirnameList = []
|
|
49
|
self.dirnameList = []
|
|
|
48
|
self.filenameList = []
|
|
50
|
self.filenameList = []
|
|
|
49
|
self.fileIndex = None
|
|
51
|
self.fileIndex = None
|
|
|
50
|
self.flagNoMoreFiles = False
|
|
52
|
self.flagNoMoreFiles = False
|
|
|
51
|
self.flagIsNewFile = 0
|
|
53
|
self.flagIsNewFile = 0
|
|
|
52
|
self.filename = ''
|
|
54
|
self.filename = ''
|
|
|
53
|
self.amisrFilePointer = None
|
|
55
|
self.amisrFilePointer = None
|
|
|
54
|
self.realBeamCode = []
|
|
56
|
self.realBeamCode = []
|
|
|
55
|
|
|
57
|
|
|
|
56
|
self.dataShape = None
|
|
58
|
self.dataShape = None
|
|
|
57
|
|
|
59
|
|
|
|
58
|
|
|
60
|
|
|
|
59
|
|
|
61
|
|
|
|
60
|
self.profileIndex = 0
|
|
62
|
self.profileIndex = 0
|
|
|
61
|
|
|
63
|
|
|
|
62
|
|
|
64
|
|
|
|
63
|
self.beamCodeByFrame = None
|
|
65
|
self.beamCodeByFrame = None
|
|
|
64
|
self.radacTimeByFrame = None
|
|
66
|
self.radacTimeByFrame = None
|
|
|
65
|
|
|
67
|
|
|
|
66
|
self.dataset = None
|
|
68
|
self.dataset = None
|
|
|
67
|
|
|
69
|
|
|
|
68
|
|
|
70
|
|
|
|
69
|
|
|
71
|
|
|
|
70
|
|
|
72
|
|
|
|
71
|
self.__firstFile = True
|
|
73
|
self.__firstFile = True
|
|
|
72
|
|
|
74
|
|
|
|
73
|
self.buffer = None
|
|
75
|
self.buffer = None
|
|
|
74
|
|
|
76
|
|
|
|
75
|
|
|
77
|
|
|
|
76
|
self.timezone = 'ut'
|
|
78
|
self.timezone = 'ut'
|
|
|
77
|
|
|
79
|
|
|
|
78
|
self.__waitForNewFile = 20
|
|
80
|
self.__waitForNewFile = 20
|
|
|
79
|
self.__filename_online = None
|
|
81
|
self.__filename_online = None
|
|
|
80
|
#Is really necessary create the output object in the initializer
|
|
82
|
#Is really necessary create the output object in the initializer
|
|
|
81
|
self.dataOut = Voltage()
|
|
83
|
self.dataOut = Voltage()
|
|
|
82
|
self.dataOut.error=False
|
|
84
|
self.dataOut.error=False
|
|
|
83
|
|
|
85
|
|
|
|
84
|
|
|
86
|
|
|
|
85
|
def setup(self,path=None,
|
|
87
|
def setup(self,path=None,
|
|
|
86
|
startDate=None,
|
|
88
|
startDate=None,
|
|
|
87
|
endDate=None,
|
|
89
|
endDate=None,
|
|
|
88
|
startTime=None,
|
|
90
|
startTime=None,
|
|
|
89
|
endTime=None,
|
|
91
|
endTime=None,
|
|
|
90
|
walk=True,
|
|
92
|
walk=True,
|
|
|
91
|
timezone='ut',
|
|
93
|
timezone='ut',
|
|
|
92
|
all=0,
|
|
94
|
all=0,
|
|
|
93
|
code = None,
|
|
95
|
code = None,
|
|
|
94
|
nCode = 0,
|
|
96
|
nCode = 0,
|
|
|
95
|
nBaud = 0,
|
|
97
|
nBaud = 0,
|
|
|
96
|
online=False):
|
|
98
|
online=False):
|
|
|
97
|
|
|
99
|
|
|
|
98
|
|
|
100
|
|
|
|
99
|
|
|
101
|
|
|
|
100
|
self.timezone = timezone
|
|
102
|
self.timezone = timezone
|
|
|
101
|
self.all = all
|
|
103
|
self.all = all
|
|
|
102
|
self.online = online
|
|
104
|
self.online = online
|
|
|
103
|
|
|
105
|
|
|
|
104
|
self.code = code
|
|
106
|
self.code = code
|
|
|
105
|
self.nCode = int(nCode)
|
|
107
|
self.nCode = int(nCode)
|
|
|
106
|
self.nBaud = int(nBaud)
|
|
108
|
self.nBaud = int(nBaud)
|
|
|
107
|
|
|
109
|
|
|
|
108
|
|
|
110
|
|
|
|
109
|
|
|
111
|
|
|
|
110
|
#self.findFiles()
|
|
112
|
#self.findFiles()
|
|
|
111
|
if not(online):
|
|
113
|
if not(online):
|
|
|
112
|
#Busqueda de archivos offline
|
|
114
|
#Busqueda de archivos offline
|
|
|
113
|
self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
|
|
115
|
self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
|
|
|
114
|
else:
|
|
116
|
else:
|
|
|
115
|
self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
|
|
117
|
self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
|
|
|
116
|
|
|
118
|
|
|
|
117
|
if not(self.filenameList):
|
|
119
|
if not(self.filenameList):
|
|
|
118
|
print("There is no files into the folder: %s"%(path))
|
|
120
|
print("There is no files into the folder: %s"%(path))
|
|
|
119
|
sys.exit()
|
|
121
|
sys.exit()
|
|
|
120
|
|
|
122
|
|
|
|
121
|
self.fileIndex = 0
|
|
123
|
self.fileIndex = 0
|
|
|
122
|
|
|
124
|
|
|
|
123
|
self.readNextFile(online)
|
|
125
|
self.readNextFile(online)
|
|
|
124
|
|
|
126
|
|
|
|
125
|
'''
|
|
127
|
'''
|
|
|
126
|
Add code
|
|
128
|
Add code
|
|
|
127
|
'''
|
|
129
|
'''
|
|
|
128
|
self.isConfig = True
|
|
130
|
self.isConfig = True
|
|
|
129
|
# print("Setup Done")
|
|
131
|
# print("Setup Done")
|
|
|
130
|
pass
|
|
132
|
pass
|
|
|
131
|
|
|
133
|
|
|
|
132
|
|
|
134
|
|
|
|
133
|
def readAMISRHeader(self,fp):
|
|
135
|
def readAMISRHeader(self,fp):
|
|
|
134
|
|
|
136
|
|
|
|
135
|
if self.isConfig and (not self.flagNoMoreFiles):
|
|
137
|
if self.isConfig and (not self.flagNoMoreFiles):
|
|
|
136
|
newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
|
|
138
|
newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
|
|
|
137
|
if self.dataShape != newShape and newShape != None:
|
|
139
|
if self.dataShape != newShape and newShape != None:
|
|
|
138
|
print("\nNEW FILE HAS A DIFFERENT SHAPE")
|
|
140
|
print("\nNEW FILE HAS A DIFFERENT SHAPE")
|
|
|
139
|
print(self.dataShape,newShape,"\n")
|
|
141
|
print(self.dataShape,newShape,"\n")
|
|
|
140
|
return 0
|
|
142
|
return 0
|
|
|
141
|
else:
|
|
143
|
else:
|
|
|
142
|
self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
|
|
144
|
self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
|
|
|
143
|
|
|
145
|
|
|
|
144
|
|
|
146
|
|
|
|
145
|
header = 'Raw11/Data/RadacHeader'
|
|
147
|
header = 'Raw11/Data/RadacHeader'
|
|
|
146
|
self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
|
|
148
|
self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
|
|
|
147
|
if (self.startDate> datetime.date(2021, 7, 15)): #Se cambió la forma de extracción de Apuntes el 17
|
|
149
|
if (self.startDate> datetime.date(2021, 7, 15)): #Se cambió la forma de extracción de Apuntes el 17
|
|
|
148
|
self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
|
|
150
|
self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
|
|
|
149
|
self.trueBeams = self.beamcodeFile.split("\n")
|
|
151
|
self.trueBeams = self.beamcodeFile.split("\n")
|
|
|
150
|
self.trueBeams.pop()#remove last
|
|
152
|
self.trueBeams.pop()#remove last
|
|
|
151
|
[self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
|
|
153
|
[self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
|
|
|
152
|
self.beamCode = [int(x, 16) for x in self.realBeamCode]
|
|
154
|
self.beamCode = [int(x, 16) for x in self.realBeamCode]
|
|
|
153
|
else:
|
|
155
|
else:
|
|
|
154
|
_beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
|
|
156
|
_beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
|
|
|
155
|
self.beamCode = _beamCode[0,:]
|
|
157
|
self.beamCode = _beamCode[0,:]
|
|
|
156
|
|
|
158
|
|
|
|
157
|
|
|
159
|
|
|
|
158
|
#self.code = fp.get(header+'/Code') # NOT USE FOR THIS
|
|
160
|
#self.code = fp.get(header+'/Code') # NOT USE FOR THIS
|
|
|
159
|
self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
|
|
161
|
self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
|
|
|
160
|
self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
|
|
162
|
self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
|
|
|
161
|
self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
|
|
163
|
self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
|
|
|
162
|
self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
|
|
164
|
self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
|
|
|
163
|
self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
|
|
165
|
self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
|
|
|
164
|
self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
|
|
166
|
self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
|
|
|
165
|
self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
|
|
167
|
self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
|
|
|
166
|
self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
|
|
168
|
self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
|
|
|
167
|
self.frequency = fp.get('Rx/Frequency')
|
|
169
|
self.frequency = fp.get('Rx/Frequency')
|
|
|
168
|
txAus = fp.get('Raw11/Data/Pulsewidth')
|
|
170
|
txAus = fp.get('Raw11/Data/Pulsewidth')
|
|
|
169
|
|
|
171
|
|
|
|
170
|
|
|
172
|
|
|
|
171
|
self.nblocks = self.pulseCount.shape[0] #nblocks
|
|
173
|
self.nblocks = self.pulseCount.shape[0] #nblocks
|
|
|
172
|
|
|
174
|
|
|
|
173
|
self.nprofiles = self.pulseCount.shape[1] #nprofile
|
|
175
|
self.nprofiles = self.pulseCount.shape[1] #nprofile
|
|
|
174
|
self.nsa = self.nsamplesPulse[0,0] #ngates
|
|
176
|
self.nsa = self.nsamplesPulse[0,0] #ngates
|
|
|
175
|
self.nchannels = len(self.beamCode)
|
|
177
|
self.nchannels = len(self.beamCode)
|
|
|
176
|
self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
|
|
178
|
self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
|
|
|
177
|
#self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
|
|
179
|
#self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
|
|
|
178
|
self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
|
|
180
|
self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
|
|
|
179
|
|
|
181
|
|
|
|
180
|
#filling radar controller header parameters
|
|
182
|
#filling radar controller header parameters
|
|
|
181
|
self.__ippKm = self.ippSeconds *.15*1e6 # in km
|
|
183
|
self.__ippKm = self.ippSeconds *.15*1e6 # in km
|
|
|
182
|
self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
|
|
184
|
self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
|
|
|
183
|
self.__txB = 0
|
|
185
|
self.__txB = 0
|
|
|
184
|
nWindows=1
|
|
186
|
nWindows=1
|
|
|
185
|
self.__nSamples = self.nsa
|
|
187
|
self.__nSamples = self.nsa
|
|
|
186
|
self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
|
|
188
|
self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
|
|
|
187
|
self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
|
|
189
|
self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
|
|
|
188
|
|
|
190
|
|
|
|
189
|
#for now until understand why the code saved is different (code included even though code not in tuf file)
|
|
191
|
#for now until understand why the code saved is different (code included even though code not in tuf file)
|
|
|
190
|
#self.__codeType = 0
|
|
192
|
#self.__codeType = 0
|
|
|
191
|
# self.__nCode = None
|
|
193
|
# self.__nCode = None
|
|
|
192
|
# self.__nBaud = None
|
|
194
|
# self.__nBaud = None
|
|
|
193
|
self.__code = self.code
|
|
195
|
self.__code = self.code
|
|
|
194
|
self.__codeType = 0
|
|
196
|
self.__codeType = 0
|
|
|
195
|
if self.code != None:
|
|
197
|
if self.code != None:
|
|
|
196
|
self.__codeType = 1
|
|
198
|
self.__codeType = 1
|
|
|
197
|
self.__nCode = self.nCode
|
|
199
|
self.__nCode = self.nCode
|
|
|
198
|
self.__nBaud = self.nBaud
|
|
200
|
self.__nBaud = self.nBaud
|
|
|
199
|
#self.__code = 0
|
|
201
|
#self.__code = 0
|
|
|
200
|
|
|
202
|
|
|
|
201
|
#filling system header parameters
|
|
203
|
#filling system header parameters
|
|
|
202
|
self.__nSamples = self.nsa
|
|
204
|
self.__nSamples = self.nsa
|
|
|
203
|
self.newProfiles = self.nprofiles/self.nchannels
|
|
205
|
self.newProfiles = self.nprofiles/self.nchannels
|
|
|
204
|
self.__channelList = list(range(self.nchannels))
|
|
206
|
self.__channelList = list(range(self.nchannels))
|
|
|
205
|
|
|
207
|
|
|
|
206
|
self.__frequency = self.frequency[0][0]
|
|
208
|
self.__frequency = self.frequency[0][0]
|
|
|
207
|
|
|
209
|
|
|
|
208
|
|
|
210
|
|
|
|
209
|
return 1
|
|
211
|
return 1
|
|
|
210
|
|
|
212
|
|
|
|
211
|
|
|
213
|
|
|
|
212
|
def createBuffers(self):
|
|
214
|
def createBuffers(self):
|
|
|
213
|
|
|
215
|
|
|
|
214
|
pass
|
|
216
|
pass
|
|
|
215
|
|
|
217
|
|
|
|
216
|
def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
|
|
218
|
def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
|
|
|
217
|
self.path = path
|
|
219
|
self.path = path
|
|
|
218
|
self.startDate = startDate
|
|
220
|
self.startDate = startDate
|
|
|
219
|
self.endDate = endDate
|
|
221
|
self.endDate = endDate
|
|
|
220
|
self.startTime = startTime
|
|
222
|
self.startTime = startTime
|
|
|
221
|
self.endTime = endTime
|
|
223
|
self.endTime = endTime
|
|
|
222
|
self.walk = walk
|
|
224
|
self.walk = walk
|
|
|
223
|
|
|
225
|
|
|
|
224
|
def __checkPath(self):
|
|
226
|
def __checkPath(self):
|
|
|
225
|
if os.path.exists(self.path):
|
|
227
|
if os.path.exists(self.path):
|
|
|
226
|
self.status = 1
|
|
228
|
self.status = 1
|
|
|
227
|
else:
|
|
229
|
else:
|
|
|
228
|
self.status = 0
|
|
230
|
self.status = 0
|
|
|
229
|
print('Path:%s does not exists'%self.path)
|
|
231
|
print('Path:%s does not exists'%self.path)
|
|
|
230
|
|
|
232
|
|
|
|
231
|
return
|
|
233
|
return
|
|
|
232
|
|
|
234
|
|
|
|
233
|
|
|
235
|
|
|
|
234
|
def __selDates(self, amisr_dirname_format):
|
|
236
|
def __selDates(self, amisr_dirname_format):
|
|
|
235
|
try:
|
|
237
|
try:
|
|
|
236
|
year = int(amisr_dirname_format[0:4])
|
|
238
|
year = int(amisr_dirname_format[0:4])
|
|
|
237
|
month = int(amisr_dirname_format[4:6])
|
|
239
|
month = int(amisr_dirname_format[4:6])
|
|
|
238
|
dom = int(amisr_dirname_format[6:8])
|
|
240
|
dom = int(amisr_dirname_format[6:8])
|
|
|
239
|
thisDate = datetime.date(year,month,dom)
|
|
241
|
thisDate = datetime.date(year,month,dom)
|
|
|
240
|
|
|
242
|
|
|
|
241
|
if (thisDate>=self.startDate and thisDate <= self.endDate):
|
|
243
|
if (thisDate>=self.startDate and thisDate <= self.endDate):
|
|
|
242
|
return amisr_dirname_format
|
|
244
|
return amisr_dirname_format
|
|
|
243
|
except:
|
|
245
|
except:
|
|
|
244
|
return None
|
|
246
|
return None
|
|
|
245
|
|
|
247
|
|
|
|
246
|
|
|
248
|
|
|
|
247
|
def __findDataForDates(self,online=False):
|
|
249
|
def __findDataForDates(self,online=False):
|
|
|
248
|
|
|
250
|
|
|
|
249
|
if not(self.status):
|
|
251
|
if not(self.status):
|
|
|
250
|
return None
|
|
252
|
return None
|
|
|
251
|
|
|
253
|
|
|
|
252
|
pat = '\d+.\d+'
|
|
254
|
pat = '\d+.\d+'
|
|
|
253
|
dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
|
|
255
|
dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
|
|
|
254
|
dirnameList = [x for x in dirnameList if x!=None]
|
|
256
|
dirnameList = [x for x in dirnameList if x!=None]
|
|
|
255
|
dirnameList = [x.string for x in dirnameList]
|
|
257
|
dirnameList = [x.string for x in dirnameList]
|
|
|
256
|
if not(online):
|
|
258
|
if not(online):
|
|
|
257
|
dirnameList = [self.__selDates(x) for x in dirnameList]
|
|
259
|
dirnameList = [self.__selDates(x) for x in dirnameList]
|
|
|
258
|
dirnameList = [x for x in dirnameList if x!=None]
|
|
260
|
dirnameList = [x for x in dirnameList if x!=None]
|
|
|
259
|
if len(dirnameList)>0:
|
|
261
|
if len(dirnameList)>0:
|
|
|
260
|
self.status = 1
|
|
262
|
self.status = 1
|
|
|
261
|
self.dirnameList = dirnameList
|
|
263
|
self.dirnameList = dirnameList
|
|
|
262
|
self.dirnameList.sort()
|
|
264
|
self.dirnameList.sort()
|
|
|
263
|
else:
|
|
265
|
else:
|
|
|
264
|
self.status = 0
|
|
266
|
self.status = 0
|
|
|
265
|
return None
|
|
267
|
return None
|
|
|
266
|
|
|
268
|
|
|
|
267
|
def __getTimeFromData(self):
|
|
269
|
def __getTimeFromData(self):
|
|
|
268
|
startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
|
|
270
|
startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
|
|
|
269
|
endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
|
|
271
|
endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
|
|
|
270
|
|
|
272
|
|
|
|
271
|
print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
|
|
273
|
print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
|
|
|
272
|
print('........................................')
|
|
274
|
print('........................................')
|
|
|
273
|
filter_filenameList = []
|
|
275
|
filter_filenameList = []
|
|
|
274
|
self.filenameList.sort()
|
|
276
|
self.filenameList.sort()
|
|
|
275
|
#for i in range(len(self.filenameList)-1):
|
|
277
|
#for i in range(len(self.filenameList)-1):
|
|
|
276
|
for i in range(len(self.filenameList)):
|
|
278
|
for i in range(len(self.filenameList)):
|
|
|
277
|
filename = self.filenameList[i]
|
|
279
|
filename = self.filenameList[i]
|
|
|
278
|
fp = h5py.File(filename,'r')
|
|
280
|
fp = h5py.File(filename,'r')
|
|
|
279
|
time_str = fp.get('Time/RadacTimeString')
|
|
281
|
time_str = fp.get('Time/RadacTimeString')
|
|
|
280
|
|
|
282
|
|
|
|
281
|
startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
|
|
283
|
startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
|
|
|
282
|
#startDateTimeStr_File = "2019-12-16 09:21:11"
|
|
284
|
#startDateTimeStr_File = "2019-12-16 09:21:11"
|
|
|
283
|
junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
|
|
285
|
junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
|
|
|
284
|
startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
|
|
286
|
startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
|
|
|
285
|
|
|
287
|
|
|
|
286
|
#endDateTimeStr_File = "2019-12-16 11:10:11"
|
|
288
|
#endDateTimeStr_File = "2019-12-16 11:10:11"
|
|
|
287
|
endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
|
|
289
|
endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
|
|
|
288
|
junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
|
|
290
|
junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
|
|
|
289
|
endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
|
|
291
|
endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
|
|
|
290
|
|
|
292
|
|
|
|
291
|
fp.close()
|
|
293
|
fp.close()
|
|
|
292
|
|
|
294
|
|
|
|
293
|
#print("check time", startDateTime_File)
|
|
295
|
#print("check time", startDateTime_File)
|
|
|
294
|
if self.timezone == 'lt':
|
|
296
|
if self.timezone == 'lt':
|
|
|
295
|
startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
|
|
297
|
startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
|
|
|
296
|
endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
|
|
298
|
endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
|
|
|
297
|
if (endDateTime_File>=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
|
|
299
|
if (endDateTime_File>=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
|
|
|
298
|
filter_filenameList.append(filename)
|
|
300
|
filter_filenameList.append(filename)
|
|
|
299
|
|
|
301
|
|
|
|
300
|
if (endDateTime_File>endDateTime_Reader):
|
|
302
|
if (endDateTime_File>endDateTime_Reader):
|
|
|
301
|
break
|
|
303
|
break
|
|
|
302
|
|
|
304
|
|
|
|
303
|
|
|
305
|
|
|
|
304
|
filter_filenameList.sort()
|
|
306
|
filter_filenameList.sort()
|
|
|
305
|
self.filenameList = filter_filenameList
|
|
307
|
self.filenameList = filter_filenameList
|
|
|
306
|
return 1
|
|
308
|
return 1
|
|
|
307
|
|
|
309
|
|
|
|
308
|
def __filterByGlob1(self, dirName):
|
|
310
|
def __filterByGlob1(self, dirName):
|
|
|
309
|
filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
|
|
311
|
filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
|
|
|
310
|
filter_files.sort()
|
|
312
|
filter_files.sort()
|
|
|
311
|
filterDict = {}
|
|
313
|
filterDict = {}
|
|
|
312
|
filterDict.setdefault(dirName)
|
|
314
|
filterDict.setdefault(dirName)
|
|
|
313
|
filterDict[dirName] = filter_files
|
|
315
|
filterDict[dirName] = filter_files
|
|
|
314
|
return filterDict
|
|
316
|
return filterDict
|
|
|
315
|
|
|
317
|
|
|
|
316
|
def __getFilenameList(self, fileListInKeys, dirList):
|
|
318
|
def __getFilenameList(self, fileListInKeys, dirList):
|
|
|
317
|
for value in fileListInKeys:
|
|
319
|
for value in fileListInKeys:
|
|
|
318
|
dirName = list(value.keys())[0]
|
|
320
|
dirName = list(value.keys())[0]
|
|
|
319
|
for file in value[dirName]:
|
|
321
|
for file in value[dirName]:
|
|
|
320
|
filename = os.path.join(dirName, file)
|
|
322
|
filename = os.path.join(dirName, file)
|
|
|
321
|
self.filenameList.append(filename)
|
|
323
|
self.filenameList.append(filename)
|
|
|
322
|
|
|
324
|
|
|
|
323
|
|
|
325
|
|
|
|
324
|
def __selectDataForTimes(self, online=False):
|
|
326
|
def __selectDataForTimes(self, online=False):
|
|
|
325
|
#aun no esta implementado el filtro for tiempo
|
|
327
|
#aun no esta implementado el filtro for tiempo
|
|
|
326
|
if not(self.status):
|
|
328
|
if not(self.status):
|
|
|
327
|
return None
|
|
329
|
return None
|
|
|
328
|
|
|
330
|
|
|
|
329
|
dirList = [os.path.join(self.path,x) for x in self.dirnameList]
|
|
331
|
dirList = [os.path.join(self.path,x) for x in self.dirnameList]
|
|
|
330
|
|
|
332
|
|
|
|
331
|
fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
|
|
333
|
fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
|
|
|
332
|
|
|
334
|
|
|
|
333
|
self.__getFilenameList(fileListInKeys, dirList)
|
|
335
|
self.__getFilenameList(fileListInKeys, dirList)
|
|
|
334
|
if not(online):
|
|
336
|
if not(online):
|
|
|
335
|
#filtro por tiempo
|
|
337
|
#filtro por tiempo
|
|
|
336
|
if not(self.all):
|
|
338
|
if not(self.all):
|
|
|
337
|
self.__getTimeFromData()
|
|
339
|
self.__getTimeFromData()
|
|
|
338
|
|
|
340
|
|
|
|
339
|
if len(self.filenameList)>0:
|
|
341
|
if len(self.filenameList)>0:
|
|
|
340
|
self.status = 1
|
|
342
|
self.status = 1
|
|
|
341
|
self.filenameList.sort()
|
|
343
|
self.filenameList.sort()
|
|
|
342
|
else:
|
|
344
|
else:
|
|
|
343
|
self.status = 0
|
|
345
|
self.status = 0
|
|
|
344
|
return None
|
|
346
|
return None
|
|
|
345
|
|
|
347
|
|
|
|
346
|
else:
|
|
348
|
else:
|
|
|
347
|
#get the last file - 1
|
|
349
|
#get the last file - 1
|
|
|
348
|
self.filenameList = [self.filenameList[-1]]
|
|
350
|
self.filenameList = [self.filenameList[-1]]
|
|
|
349
|
new_dirnameList = []
|
|
351
|
new_dirnameList = []
|
|
|
350
|
for dirname in self.dirnameList:
|
|
352
|
for dirname in self.dirnameList:
|
|
|
351
|
junk = numpy.array([dirname in x for x in self.filenameList])
|
|
353
|
junk = numpy.array([dirname in x for x in self.filenameList])
|
|
|
352
|
junk_sum = junk.sum()
|
|
354
|
junk_sum = junk.sum()
|
|
|
353
|
if junk_sum > 0:
|
|
355
|
if junk_sum > 0:
|
|
|
354
|
new_dirnameList.append(dirname)
|
|
356
|
new_dirnameList.append(dirname)
|
|
|
355
|
self.dirnameList = new_dirnameList
|
|
357
|
self.dirnameList = new_dirnameList
|
|
|
356
|
return 1
|
|
358
|
return 1
|
|
|
357
|
|
|
359
|
|
|
|
358
|
def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
|
|
360
|
def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
|
|
|
359
|
endTime=datetime.time(23,59,59),walk=True):
|
|
361
|
endTime=datetime.time(23,59,59),walk=True):
|
|
|
360
|
|
|
362
|
|
|
|
361
|
if endDate ==None:
|
|
363
|
if endDate ==None:
|
|
|
362
|
startDate = datetime.datetime.utcnow().date()
|
|
364
|
startDate = datetime.datetime.utcnow().date()
|
|
|
363
|
endDate = datetime.datetime.utcnow().date()
|
|
365
|
endDate = datetime.datetime.utcnow().date()
|
|
|
364
|
|
|
366
|
|
|
|
365
|
self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
|
|
367
|
self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
|
|
|
366
|
|
|
368
|
|
|
|
367
|
self.__checkPath()
|
|
369
|
self.__checkPath()
|
|
|
368
|
|
|
370
|
|
|
|
369
|
self.__findDataForDates(online=True)
|
|
371
|
self.__findDataForDates(online=True)
|
|
|
370
|
|
|
372
|
|
|
|
371
|
self.dirnameList = [self.dirnameList[-1]]
|
|
373
|
self.dirnameList = [self.dirnameList[-1]]
|
|
|
372
|
|
|
374
|
|
|
|
373
|
self.__selectDataForTimes(online=True)
|
|
375
|
self.__selectDataForTimes(online=True)
|
|
|
374
|
|
|
376
|
|
|
|
375
|
return
|
|
377
|
return
|
|
|
376
|
|
|
378
|
|
|
|
377
|
|
|
379
|
|
|
|
378
|
def searchFilesOffLine(self,
|
|
380
|
def searchFilesOffLine(self,
|
|
|
379
|
path,
|
|
381
|
path,
|
|
|
380
|
startDate,
|
|
382
|
startDate,
|
|
|
381
|
endDate,
|
|
383
|
endDate,
|
|
|
382
|
startTime=datetime.time(0,0,0),
|
|
384
|
startTime=datetime.time(0,0,0),
|
|
|
383
|
endTime=datetime.time(23,59,59),
|
|
385
|
endTime=datetime.time(23,59,59),
|
|
|
384
|
walk=True):
|
|
386
|
walk=True):
|
|
|
385
|
|
|
387
|
|
|
|
386
|
self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
|
|
388
|
self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
|
|
|
387
|
|
|
389
|
|
|
|
388
|
self.__checkPath()
|
|
390
|
self.__checkPath()
|
|
|
389
|
|
|
391
|
|
|
|
390
|
self.__findDataForDates()
|
|
392
|
self.__findDataForDates()
|
|
|
391
|
|
|
393
|
|
|
|
392
|
self.__selectDataForTimes()
|
|
394
|
self.__selectDataForTimes()
|
|
|
393
|
|
|
395
|
|
|
|
394
|
for i in range(len(self.filenameList)):
|
|
396
|
for i in range(len(self.filenameList)):
|
|
|
395
|
print("%s" %(self.filenameList[i]))
|
|
397
|
print("%s" %(self.filenameList[i]))
|
|
|
396
|
|
|
398
|
|
|
|
397
|
return
|
|
399
|
return
|
|
|
398
|
|
|
400
|
|
|
|
399
|
def __setNextFileOffline(self):
|
|
401
|
def __setNextFileOffline(self):
|
|
|
400
|
|
|
402
|
|
|
|
401
|
try:
|
|
403
|
try:
|
|
|
402
|
self.filename = self.filenameList[self.fileIndex]
|
|
404
|
self.filename = self.filenameList[self.fileIndex]
|
|
|
403
|
self.amisrFilePointer = h5py.File(self.filename,'r')
|
|
405
|
self.amisrFilePointer = h5py.File(self.filename,'r')
|
|
|
404
|
self.fileIndex += 1
|
|
406
|
self.fileIndex += 1
|
|
|
405
|
except:
|
|
407
|
except:
|
|
|
406
|
self.flagNoMoreFiles = 1
|
|
408
|
self.flagNoMoreFiles = 1
|
|
|
407
|
print("No more Files")
|
|
409
|
print("No more Files")
|
|
|
408
|
return 0
|
|
410
|
return 0
|
|
|
409
|
|
|
411
|
|
|
|
410
|
self.flagIsNewFile = 1
|
|
412
|
self.flagIsNewFile = 1
|
|
|
411
|
print("Setting the file: %s"%self.filename)
|
|
413
|
print("Setting the file: %s"%self.filename)
|
|
|
412
|
|
|
414
|
|
|
|
413
|
return 1
|
|
415
|
return 1
|
|
|
414
|
|
|
416
|
|
|
|
415
|
|
|
417
|
|
|
|
416
|
def __setNextFileOnline(self):
|
|
418
|
def __setNextFileOnline(self):
|
|
|
417
|
filename = self.filenameList[0]
|
|
419
|
filename = self.filenameList[0]
|
|
|
418
|
if self.__filename_online != None:
|
|
420
|
if self.__filename_online != None:
|
|
|
419
|
self.__selectDataForTimes(online=True)
|
|
421
|
self.__selectDataForTimes(online=True)
|
|
|
420
|
filename = self.filenameList[0]
|
|
422
|
filename = self.filenameList[0]
|
|
|
421
|
wait = 0
|
|
423
|
wait = 0
|
|
|
422
|
self.__waitForNewFile=180 ## DEBUG:
|
|
424
|
#self.__waitForNewFile=180 ## DEBUG:
|
|
|
423
|
while self.__filename_online == filename:
|
|
425
|
while self.__filename_online == filename:
|
|
|
424
|
print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
|
|
426
|
print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
|
|
|
425
|
if wait == 5:
|
|
427
|
if wait == 5:
|
|
|
426
|
self.flagNoMoreFiles = 1
|
|
428
|
self.flagNoMoreFiles = 1
|
|
|
427
|
return 0
|
|
429
|
return 0
|
|
|
428
|
sleep(self.__waitForNewFile)
|
|
430
|
sleep(self.__waitForNewFile)
|
|
|
429
|
self.__selectDataForTimes(online=True)
|
|
431
|
self.__selectDataForTimes(online=True)
|
|
|
430
|
filename = self.filenameList[0]
|
|
432
|
filename = self.filenameList[0]
|
|
|
431
|
wait += 1
|
|
433
|
wait += 1
|
|
|
432
|
|
|
434
|
|
|
|
433
|
self.__filename_online = filename
|
|
435
|
self.__filename_online = filename
|
|
|
434
|
|
|
436
|
|
|
|
435
|
self.amisrFilePointer = h5py.File(filename,'r')
|
|
437
|
self.amisrFilePointer = h5py.File(filename,'r')
|
|
|
436
|
self.flagIsNewFile = 1
|
|
438
|
self.flagIsNewFile = 1
|
|
|
437
|
self.filename = filename
|
|
439
|
self.filename = filename
|
|
|
438
|
print("Setting the file: %s"%self.filename)
|
|
440
|
print("Setting the file: %s"%self.filename)
|
|
|
439
|
return 1
|
|
441
|
return 1
|
|
|
440
|
|
|
442
|
|
|
|
441
|
|
|
443
|
|
|
|
442
|
def readData(self):
|
|
444
|
def readData(self):
|
|
|
443
|
buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
|
|
445
|
buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
|
|
|
444
|
re = buffer[:,:,:,0]
|
|
446
|
re = buffer[:,:,:,0]
|
|
|
445
|
im = buffer[:,:,:,1]
|
|
447
|
im = buffer[:,:,:,1]
|
|
|
446
|
dataset = re + im*1j
|
|
448
|
dataset = re + im*1j
|
|
|
447
|
|
|
449
|
|
|
|
448
|
self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
|
|
450
|
self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
|
|
|
449
|
timeset = self.radacTime[:,0]
|
|
451
|
timeset = self.radacTime[:,0]
|
|
|
450
|
|
|
452
|
|
|
|
451
|
return dataset,timeset
|
|
453
|
return dataset,timeset
|
|
|
452
|
|
|
454
|
|
|
|
453
|
def reshapeData(self):
|
|
455
|
def reshapeData(self):
|
|
|
454
|
#self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
|
|
456
|
#self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
|
|
|
455
|
channels = self.beamCodeByPulse[0,:]
|
|
457
|
channels = self.beamCodeByPulse[0,:]
|
|
|
456
|
nchan = self.nchannels
|
|
458
|
nchan = self.nchannels
|
|
|
457
|
#self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
|
|
459
|
#self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
|
|
|
458
|
nblocks = self.nblocks
|
|
460
|
nblocks = self.nblocks
|
|
|
459
|
nsamples = self.nsa
|
|
461
|
nsamples = self.nsa
|
|
|
460
|
|
|
462
|
|
|
|
461
|
#Dimensions : nChannels, nProfiles, nSamples
|
|
463
|
#Dimensions : nChannels, nProfiles, nSamples
|
|
|
462
|
new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
|
|
464
|
new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
|
|
|
463
|
############################################
|
|
465
|
############################################
|
|
|
464
|
|
|
466
|
|
|
|
465
|
for thisChannel in range(nchan):
|
|
467
|
for thisChannel in range(nchan):
|
|
|
466
|
new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
|
|
468
|
new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
|
|
|
467
|
|
|
469
|
|
|
|
468
|
|
|
470
|
|
|
|
469
|
new_block = numpy.transpose(new_block, (1,0,2,3))
|
|
471
|
new_block = numpy.transpose(new_block, (1,0,2,3))
|
|
|
470
|
new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
|
|
472
|
new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
|
|
|
471
|
|
|
473
|
|
|
|
472
|
return new_block
|
|
474
|
return new_block
|
|
|
473
|
|
|
475
|
|
|
|
474
|
def updateIndexes(self):
|
|
476
|
def updateIndexes(self):
|
|
|
475
|
|
|
477
|
|
|
|
476
|
pass
|
|
478
|
pass
|
|
|
477
|
|
|
479
|
|
|
|
478
|
def fillJROHeader(self):
|
|
480
|
def fillJROHeader(self):
|
|
|
479
|
|
|
481
|
|
|
|
480
|
#fill radar controller header
|
|
482
|
#fill radar controller header
|
|
|
481
|
self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
|
|
483
|
self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
|
|
|
482
|
txA=self.__txA,
|
|
484
|
txA=self.__txA,
|
|
|
483
|
txB=0,
|
|
485
|
txB=0,
|
|
|
484
|
nWindows=1,
|
|
486
|
nWindows=1,
|
|
|
485
|
nHeights=self.__nSamples,
|
|
487
|
nHeights=self.__nSamples,
|
|
|
486
|
firstHeight=self.__firstHeight,
|
|
488
|
firstHeight=self.__firstHeight,
|
|
|
487
|
deltaHeight=self.__deltaHeight,
|
|
489
|
deltaHeight=self.__deltaHeight,
|
|
|
488
|
codeType=self.__codeType,
|
|
490
|
codeType=self.__codeType,
|
|
|
489
|
nCode=self.__nCode, nBaud=self.__nBaud,
|
|
491
|
nCode=self.__nCode, nBaud=self.__nBaud,
|
|
|
490
|
code = self.__code,
|
|
492
|
code = self.__code,
|
|
|
491
|
fClock=1)
|
|
493
|
fClock=1)
|
|
|
492
|
|
|
494
|
|
|
|
493
|
#fill system header
|
|
495
|
#fill system header
|
|
|
494
|
self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
|
|
496
|
self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
|
|
|
495
|
nProfiles=self.newProfiles,
|
|
497
|
nProfiles=self.newProfiles,
|
|
|
496
|
nChannels=len(self.__channelList),
|
|
498
|
nChannels=len(self.__channelList),
|
|
|
497
|
adcResolution=14,
|
|
499
|
adcResolution=14,
|
|
|
498
|
pciDioBusWidth=32)
|
|
500
|
pciDioBusWidth=32)
|
|
|
499
|
|
|
501
|
|
|
|
500
|
self.dataOut.type = "Voltage"
|
|
502
|
self.dataOut.type = "Voltage"
|
|
|
501
|
|
|
503
|
|
|
|
502
|
self.dataOut.data = None
|
|
504
|
self.dataOut.data = None
|
|
|
503
|
|
|
505
|
|
|
|
504
|
self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
|
|
506
|
self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
|
|
|
505
|
|
|
507
|
|
|
|
506
|
# self.dataOut.nChannels = 0
|
|
508
|
# self.dataOut.nChannels = 0
|
|
|
507
|
|
|
509
|
|
|
|
508
|
# self.dataOut.nHeights = 0
|
|
510
|
# self.dataOut.nHeights = 0
|
|
|
509
|
|
|
511
|
|
|
|
510
|
self.dataOut.nProfiles = self.newProfiles*self.nblocks
|
|
512
|
self.dataOut.nProfiles = self.newProfiles*self.nblocks
|
|
|
511
|
|
|
513
|
|
|
|
512
|
#self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
|
|
514
|
#self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
|
|
|
513
|
ranges = numpy.reshape(self.rangeFromFile.value,(-1))
|
|
515
|
ranges = numpy.reshape(self.rangeFromFile.value,(-1))
|
|
|
514
|
self.dataOut.heightList = ranges/1000.0 #km
|
|
516
|
self.dataOut.heightList = ranges/1000.0 #km
|
|
|
515
|
|
|
517
|
|
|
|
516
|
|
|
518
|
|
|
|
517
|
self.dataOut.channelList = self.__channelList
|
|
519
|
self.dataOut.channelList = self.__channelList
|
|
|
518
|
|
|
520
|
|
|
|
519
|
self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
|
|
521
|
self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
|
|
|
520
|
|
|
522
|
|
|
|
521
|
# self.dataOut.channelIndexList = None
|
|
523
|
# self.dataOut.channelIndexList = None
|
|
|
522
|
|
|
524
|
|
|
|
523
|
self.dataOut.flagNoData = True
|
|
525
|
self.dataOut.flagNoData = True
|
|
|
524
|
|
|
526
|
|
|
|
525
|
#Set to TRUE if the data is discontinuous
|
|
527
|
#Set to TRUE if the data is discontinuous
|
|
|
526
|
self.dataOut.flagDiscontinuousBlock = False
|
|
528
|
self.dataOut.flagDiscontinuousBlock = False
|
|
|
527
|
|
|
529
|
|
|
|
528
|
self.dataOut.utctime = None
|
|
530
|
self.dataOut.utctime = None
|
|
|
529
|
|
|
531
|
|
|
|
530
|
#self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
|
|
532
|
#self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
|
|
|
531
|
if self.timezone == 'lt':
|
|
533
|
if self.timezone == 'lt':
|
|
|
532
|
self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
|
|
534
|
self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
|
|
|
533
|
else:
|
|
535
|
else:
|
|
|
534
|
self.dataOut.timeZone = 0 #by default time is UTC
|
|
536
|
self.dataOut.timeZone = 0 #by default time is UTC
|
|
|
535
|
|
|
537
|
|
|
|
536
|
self.dataOut.dstFlag = 0
|
|
538
|
self.dataOut.dstFlag = 0
|
|
|
537
|
|
|
539
|
|
|
|
538
|
self.dataOut.errorCount = 0
|
|
540
|
self.dataOut.errorCount = 0
|
|
|
539
|
|
|
541
|
|
|
|
540
|
self.dataOut.nCohInt = 1
|
|
542
|
self.dataOut.nCohInt = 1
|
|
|
541
|
|
|
543
|
|
|
|
542
|
self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
|
|
544
|
self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
|
|
|
543
|
|
|
545
|
|
|
|
544
|
self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
|
|
546
|
self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
|
|
|
545
|
|
|
547
|
|
|
|
546
|
self.dataOut.flagShiftFFT = False
|
|
548
|
self.dataOut.flagShiftFFT = False
|
|
|
547
|
|
|
549
|
|
|
|
548
|
self.dataOut.ippSeconds = self.ippSeconds
|
|
550
|
self.dataOut.ippSeconds = self.ippSeconds
|
|
|
549
|
|
|
551
|
|
|
|
550
|
#Time interval between profiles
|
|
552
|
#Time interval between profiles
|
|
|
551
|
#self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
|
|
553
|
#self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
|
|
|
552
|
|
|
554
|
|
|
|
553
|
self.dataOut.frequency = self.__frequency
|
|
555
|
self.dataOut.frequency = self.__frequency
|
|
|
554
|
self.dataOut.realtime = self.online
|
|
556
|
self.dataOut.realtime = self.online
|
|
|
555
|
pass
|
|
557
|
pass
|
|
|
556
|
|
|
558
|
|
|
|
557
|
def readNextFile(self,online=False):
|
|
559
|
def readNextFile(self,online=False):
|
|
|
558
|
|
|
560
|
|
|
|
559
|
if not(online):
|
|
561
|
if not(online):
|
|
|
560
|
newFile = self.__setNextFileOffline()
|
|
562
|
newFile = self.__setNextFileOffline()
|
|
|
561
|
else:
|
|
563
|
else:
|
|
|
562
|
newFile = self.__setNextFileOnline()
|
|
564
|
newFile = self.__setNextFileOnline()
|
|
|
563
|
|
|
565
|
|
|
|
564
|
if not(newFile):
|
|
566
|
if not(newFile):
|
|
|
565
|
self.dataOut.error = True
|
|
567
|
self.dataOut.error = True
|
|
|
566
|
return 0
|
|
568
|
return 0
|
|
|
567
|
|
|
569
|
|
|
|
568
|
if not self.readAMISRHeader(self.amisrFilePointer):
|
|
570
|
if not self.readAMISRHeader(self.amisrFilePointer):
|
|
|
569
|
self.dataOut.error = True
|
|
571
|
self.dataOut.error = True
|
|
|
570
|
return 0
|
|
572
|
return 0
|
|
|
571
|
|
|
573
|
|
|
|
572
|
self.createBuffers()
|
|
574
|
self.createBuffers()
|
|
|
573
|
self.fillJROHeader()
|
|
575
|
self.fillJROHeader()
|
|
|
574
|
|
|
576
|
|
|
|
575
|
#self.__firstFile = False
|
|
577
|
#self.__firstFile = False
|
|
|
576
|
|
|
578
|
|
|
|
577
|
|
|
579
|
|
|
|
578
|
|
|
580
|
|
|
|
579
|
self.dataset,self.timeset = self.readData()
|
|
581
|
self.dataset,self.timeset = self.readData()
|
|
|
580
|
|
|
582
|
|
|
|
581
|
if self.endDate!=None:
|
|
583
|
if self.endDate!=None:
|
|
|
582
|
endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
|
|
584
|
endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
|
|
|
583
|
time_str = self.amisrFilePointer.get('Time/RadacTimeString')
|
|
585
|
time_str = self.amisrFilePointer.get('Time/RadacTimeString')
|
|
|
584
|
startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
|
|
586
|
startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
|
|
|
585
|
junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
|
|
587
|
junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
|
|
|
586
|
startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
|
|
588
|
startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
|
|
|
587
|
if self.timezone == 'lt':
|
|
589
|
if self.timezone == 'lt':
|
|
|
588
|
startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
|
|
590
|
startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
|
|
|
589
|
if (startDateTime_File>endDateTime_Reader):
|
|
591
|
if (startDateTime_File>endDateTime_Reader):
|
|
|
590
|
return 0
|
|
592
|
return 0
|
|
|
591
|
|
|
593
|
|
|
|
592
|
self.jrodataset = self.reshapeData()
|
|
594
|
self.jrodataset = self.reshapeData()
|
|
|
593
|
#----self.updateIndexes()
|
|
595
|
#----self.updateIndexes()
|
|
|
594
|
self.profileIndex = 0
|
|
596
|
self.profileIndex = 0
|
|
|
595
|
|
|
597
|
|
|
|
596
|
return 1
|
|
598
|
return 1
|
|
|
597
|
|
|
599
|
|
|
|
598
|
|
|
600
|
|
|
|
599
|
def __hasNotDataInBuffer(self):
|
|
601
|
def __hasNotDataInBuffer(self):
|
|
|
600
|
if self.profileIndex >= (self.newProfiles*self.nblocks):
|
|
602
|
if self.profileIndex >= (self.newProfiles*self.nblocks):
|
|
|
601
|
return 1
|
|
603
|
return 1
|
|
|
602
|
return 0
|
|
604
|
return 0
|
|
|
603
|
|
|
605
|
|
|
|
604
|
|
|
606
|
|
|
|
605
|
def getData(self):
|
|
607
|
def getData(self):
|
|
|
606
|
|
|
608
|
|
|
|
607
|
if self.flagNoMoreFiles:
|
|
609
|
if self.flagNoMoreFiles:
|
|
|
608
|
self.dataOut.flagNoData = True
|
|
610
|
self.dataOut.flagNoData = True
|
|
|
609
|
return 0
|
|
611
|
return 0
|
|
|
610
|
|
|
612
|
|
|
|
611
|
if self.__hasNotDataInBuffer():
|
|
613
|
if self.__hasNotDataInBuffer():
|
|
|
612
|
if not (self.readNextFile(self.online)):
|
|
614
|
if not (self.readNextFile(self.online)):
|
|
|
613
|
return 0
|
|
615
|
return 0
|
|
|
614
|
|
|
616
|
|
|
|
615
|
|
|
617
|
|
|
|
616
|
if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
|
|
618
|
if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
|
|
|
617
|
self.dataOut.flagNoData = True
|
|
619
|
self.dataOut.flagNoData = True
|
|
|
618
|
return 0
|
|
620
|
return 0
|
|
|
619
|
|
|
621
|
|
|
|
620
|
#self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
|
|
622
|
#self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
|
|
|
621
|
|
|
623
|
|
|
|
622
|
self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
|
|
624
|
self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
|
|
|
623
|
|
|
625
|
|
|
|
624
|
#print("R_t",self.timeset)
|
|
626
|
#print("R_t",self.timeset)
|
|
|
625
|
|
|
627
|
|
|
|
626
|
#self.dataOut.utctime = self.jrotimeset[self.profileIndex]
|
|
628
|
#self.dataOut.utctime = self.jrotimeset[self.profileIndex]
|
|
|
627
|
#verificar basic header de jro data y ver si es compatible con este valor
|
|
629
|
#verificar basic header de jro data y ver si es compatible con este valor
|
|
|
628
|
#self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
|
|
630
|
#self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
|
|
|
629
|
indexprof = numpy.mod(self.profileIndex, self.newProfiles)
|
|
631
|
indexprof = numpy.mod(self.profileIndex, self.newProfiles)
|
|
|
630
|
indexblock = self.profileIndex/self.newProfiles
|
|
632
|
indexblock = self.profileIndex/self.newProfiles
|
|
|
631
|
#print (indexblock, indexprof)
|
|
633
|
#print (indexblock, indexprof)
|
|
|
632
|
diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
|
|
634
|
diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
|
|
|
633
|
diffUTC = 0
|
|
635
|
diffUTC = 0
|
|
|
634
|
t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
|
|
636
|
t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
|
|
|
635
|
|
|
637
|
|
|
|
636
|
#print("utc :",indexblock," __ ",t_comp)
|
|
638
|
#print("utc :",indexblock," __ ",t_comp)
|
|
|
637
|
#print(numpy.shape(self.timeset))
|
|
639
|
#print(numpy.shape(self.timeset))
|
|
|
638
|
self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
|
|
640
|
self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
|
|
|
639
|
#self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
|
|
641
|
#self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
|
|
|
640
|
#print(self.dataOut.utctime)
|
|
642
|
#print(self.dataOut.utctime)
|
|
|
641
|
self.dataOut.profileIndex = self.profileIndex
|
|
643
|
self.dataOut.profileIndex = self.profileIndex
|
|
|
642
|
self.dataOut.flagNoData = False
|
|
644
|
self.dataOut.flagNoData = False
|
|
|
643
|
# if indexprof == 0:
|
|
645
|
# if indexprof == 0:
|
|
|
644
|
# print self.dataOut.utctime
|
|
646
|
# print self.dataOut.utctime
|
|
|
645
|
|
|
647
|
|
|
|
646
|
self.profileIndex += 1
|
|
648
|
self.profileIndex += 1
|
|
|
647
|
|
|
649
|
|
|
|
648
|
return self.dataOut.data
|
|
650
|
return self.dataOut.data
|
|
|
649
|
|
|
651
|
|
|
|
650
|
|
|
652
|
|
|
|
651
|
def run(self, **kwargs):
|
|
653
|
def run(self, **kwargs):
|
|
|
652
|
'''
|
|
654
|
'''
|
|
|
653
|
This method will be called many times so here you should put all your code
|
|
655
|
This method will be called many times so here you should put all your code
|
|
|
654
|
'''
|
|
656
|
'''
|
|
|
655
|
#print("running kamisr")
|
|
657
|
#print("running kamisr")
|
|
|
656
|
if not self.isConfig:
|
|
658
|
if not self.isConfig:
|
|
|
657
|
self.setup(**kwargs)
|
|
659
|
self.setup(**kwargs)
|
|
|
658
|
self.isConfig = True
|
|
660
|
self.isConfig = True
|
|
|
659
|
|
|
661
|
|
|
|
660
|
self.getData()
|
|
662
|
self.getData()
|
