##// END OF EJS Templates
schain
RSS
Separación de datos antes del 17 de Julio del 2021, donde se modificó los apuntes y la lectura de estos
joabAM -
r1371:603b419b2641
parent child
Show More
Side by Side Unified
Context file:
r1371:603b419b2641 Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 <Project description="AMISR EEJ Experiment" id="11" name="Process (eej_proc)"><ReadUnit id="111" inputId="None" name="AMISRReader"><Parameter name="path" value="/media/soporte/UARS_4T_D02/AMISR_DATA/2021/" /><Parameter name="startDate" value="2021/07/11" /><Parameter name="endDate" value="2021/07/11" /><Parameter name="startTime" value="07:01:30" /><Parameter name="endTime" value="19:00:00" /><Parameter name="walk" value="1" /><Parameter name="code" value="(1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1)" /><Parameter name="nCode" value="1" /><Parameter name="nBaud" value="28" /><Parameter name="timezone" value="ut" /><Parameter name="online" value="0" /></ReadUnit><ProcUnit id="112" inputId="111" name="VoltageProc"><Operation id="1121" name="setAttribute"><Parameter name="frequency" value="445090000.0" /></Operation><Operation id="1122" name="Decoder"><Parameter name="code" value="(1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1)" /><Parameter name="nCode" value="1" /><Parameter name="nBaud" value="28" /><Parameter name="osamp" value="1" /></Operation></ProcUnit><ProcUnit id="113" inputId="112" name="SpectraProc"><Parameter name="nFFTPoints" value="16" /><Operation id="1131" name="IncohInt"><Parameter name="n" value="150" /></Operation><Operation id="1132" name="removeDC" /><Operation id="1133" name="SpectraPlot"><Parameter name="id" value="21" /><Parameter name="xaxis" value="velocity" /><Parameter name="ymax" value="300" /><Parameter name="showprofile" value="1" /><Parameter name="wintitle" value="AMISR Beam 0" /><Parameter name="zmin" value="45" /><Parameter name="zmax" value="65" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192/plots" /><Parameter name="colormap" value="jet" /><Parameter name="localtime" value="0" /><Parameter name="show" value="1" /></Operation><Operation id="1134" name="SpectraWriter"><Parameter name="path" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="blocksPerFile" value="10" /></Operation><Operation id="1135" name="NoisePlot"><Parameter name="id" value="3" /><Parameter name="wintitle" value="title0" /><Parameter name="showprofile" value="0" /><Parameter name="xmin" value="07" /><Parameter name="xmax" value="18" /><Parameter name="ymin" value="45" /><Parameter name="ymax" value="65" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="localtime" value="0" /><Parameter name="show" value="0" /></Operation><Operation id="1136" name="RTIPlot"><Parameter name="id" value="2" /><Parameter name="localtime" value="0" /><Parameter name="wintitle" value="RTI" /><Parameter name="xmin" value="07" /><Parameter name="xmax" value="18" /><Parameter name="ymin" value="0" /><Parameter name="zmin" value="45" /><Parameter name="zmax" value="65" /><Parameter name="showprofile" value="0" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192/plots" /><Parameter name="colormap" value="jet" /><Parameter name="show" value="1" /></Operation></ProcUnit><ProcUnit id="114" inputId="113" name="ParametersProc"><Operation id="1141" name="SpectralMoments" /><Operation id="1142" name="ParamWriter"><Parameter name="path" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="blocksPerFile" value="10" /><Parameter name="metadataList" value="['type', 'inputUnit', 'heightList']" /><Parameter name="dataList" value="['moments', 'data_SNR', 'utctime']" /><Parameter name="mode" value="1" /></Operation></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
@@ -25,265 +25,7 from schainpy.utils import log
25 25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 26 multiprocessing.set_start_method('fork')
27 27
28 DTYPES = {
29 'Voltage': '.r',
30 'Spectra': '.pdata'
31 }
32
33
34 def MPProject(project, n=cpu_count()):
35 '''
36 Project wrapper to run schain in n processes
37 '''
38
39 rconf = project.getReadUnitObj()
40 op = rconf.getOperationObj('run')
41 dt1 = op.getParameterValue('startDate')
42 dt2 = op.getParameterValue('endDate')
43 tm1 = op.getParameterValue('startTime')
44 tm2 = op.getParameterValue('endTime')
45 days = (dt2 - dt1).days
46
47 for day in range(days + 1):
48 skip = 0
49 cursor = 0
50 processes = []
51 dt = dt1 + datetime.timedelta(day)
52 dt_str = dt.strftime('%Y/%m/%d')
53 reader = JRODataReader()
54 paths, files = reader.searchFilesOffLine(path=rconf.path,
55 startDate=dt,
56 endDate=dt,
57 startTime=tm1,
58 endTime=tm2,
59 ext=DTYPES[rconf.datatype])
60 nFiles = len(files)
61 if nFiles == 0:
62 continue
63 skip = int(math.ceil(nFiles / n))
64 while nFiles > cursor * skip:
65 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
66 skip=skip)
67 p = project.clone()
68 p.start()
69 processes.append(p)
70 cursor += 1
71
72 def beforeExit(exctype, value, trace):
73 for process in processes:
74 process.terminate()
75 process.join()
76 print(traceback.print_tb(trace))
77
78 sys.excepthook = beforeExit
79
80 for process in processes:
81 process.join()
82 process.terminate()
83
84 time.sleep(3)
85
86 def wait(context):
87
88 time.sleep(1)
89 c = zmq.Context()
90 receiver = c.socket(zmq.SUB)
91 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
92 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
93 msg = receiver.recv_multipart()[1]
94 context.terminate()
95
96 class ParameterConf():
97
98 id = None
99 name = None
100 value = None
101 format = None
102
103 __formated_value = None
104
105 ELEMENTNAME = 'Parameter'
106
107 def __init__(self):
108
109 self.format = 'str'
110
111 def getElementName(self):
112
113 return self.ELEMENTNAME
114
115 def getValue(self):
116
117 value = self.value
118 format = self.format
119
120 if self.__formated_value != None:
121
122 return self.__formated_value
123
124 if format == 'obj':
125 return value
126
127 if format == 'str':
128 self.__formated_value = str(value)
129 return self.__formated_value
130
131 if value == '':
132 raise ValueError('%s: This parameter value is empty' % self.name)
133
134 if format == 'list':
135 strList = [s.strip() for s in value.split(',')]
136 self.__formated_value = strList
137
138 return self.__formated_value
139
140 if format == 'intlist':
141 '''
142 Example:
143 value = (0,1,2)
144 '''
145
146 new_value = ast.literal_eval(value)
147
148 if type(new_value) not in (tuple, list):
149 new_value = [int(new_value)]
150
151 self.__formated_value = new_value
152
153 return self.__formated_value
154
155 if format == 'floatlist':
156 '''
157 Example:
158 value = (0.5, 1.4, 2.7)
159 '''
160
161 new_value = ast.literal_eval(value)
162
163 if type(new_value) not in (tuple, list):
164 new_value = [float(new_value)]
165
166 self.__formated_value = new_value
167
168 return self.__formated_value
169
170 if format == 'date':
171 strList = value.split('/')
172 intList = [int(x) for x in strList]
173 date = datetime.date(intList[0], intList[1], intList[2])
174
175 self.__formated_value = date
176
177 return self.__formated_value
178
179 if format == 'time':
180 strList = value.split(':')
181 intList = [int(x) for x in strList]
182 time = datetime.time(intList[0], intList[1], intList[2])
183
184 self.__formated_value = time
185
186 return self.__formated_value
187
188 if format == 'pairslist':
189 '''
190 Example:
191 value = (0,1),(1,2)
192 '''
193
194 new_value = ast.literal_eval(value)
195
196 if type(new_value) not in (tuple, list):
197 raise ValueError('%s has to be a tuple or list of pairs' % value)
198
199 if type(new_value[0]) not in (tuple, list):
200 if len(new_value) != 2:
201 raise ValueError('%s has to be a tuple or list of pairs' % value)
202 new_value = [new_value]
203
204 for thisPair in new_value:
205 if len(thisPair) != 2:
206 raise ValueError('%s has to be a tuple or list of pairs' % value)
207
208 self.__formated_value = new_value
209
210 return self.__formated_value
211
212 if format == 'multilist':
213 '''
214 Example:
215 value = (0,1,2),(3,4,5)
216 '''
217 multiList = ast.literal_eval(value)
218
219 if type(multiList[0]) == int:
220 multiList = ast.literal_eval('(' + value + ')')
221
222 self.__formated_value = multiList
223
224 return self.__formated_value
225
226 if format == 'bool':
227 value = int(value)
228
229 if format == 'int':
230 value = float(value)
231
232 format_func = eval(format)
233
234 self.__formated_value = format_func(value)
235
236 return self.__formated_value
237
238 def updateId(self, new_id):
239
240 self.id = str(new_id)
241
242 def setup(self, id, name, value, format='str'):
243 self.id = str(id)
244 self.name = name
245 if format == 'obj':
246 self.value = value
247 else:
248 self.value = str(value)
249 self.format = str.lower(format)
250
251 self.getValue()
252
253 return 1
254
255 def update(self, name, value, format='str'):
256
257 self.name = name
258 self.value = str(value)
259 self.format = format
260
261 def makeXml(self, opElement):
262 if self.name not in ('queue',):
263 parmElement = SubElement(opElement, self.ELEMENTNAME)
264 parmElement.set('id', str(self.id))
265 parmElement.set('name', self.name)
266 parmElement.set('value', self.value)
267 parmElement.set('format', self.format)
268
269 def readXml(self, parmElement):
270
271 self.id = parmElement.get('id')
272 self.name = parmElement.get('name')
273 self.value = parmElement.get('value')
274 self.format = str.lower(parmElement.get('format'))
275
276 # Compatible with old signal chain version
277 if self.format == 'int' and self.name == 'idfigure':
278 self.name = 'id'
279
280 def printattr(self):
281
282 print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
283
284 class OperationConf():
285
286 ELEMENTNAME = 'Operation'
28 class ConfBase():
287 29
288 30 def __init__(self):
289 31
@@ -506,7 +248,6 class ProcUnitConf(ConfBase):
506 248 '''
507 249
508 250 className = eval(self.name)
509 #print(self.name)
510 251 kwargs = self.getKwargs()
511 252 procUnitObj = className()
512 253 procUnitObj.name = self.name
Show file before | Show file after | Hide comments
@@ -24,7 +24,7 from schainpy.model.data.jrodata import Voltage
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
25 25 from numpy import imag
26 26
27 @MPDecorator
27
28 28 class AMISRReader(ProcessingUnit):
29 29 '''
30 30 classdocs
@@ -82,6 +82,7 class AMISRReader(ProcessingUnit):
82 82 self.dataOut = Voltage()
83 83 self.dataOut.error=False
84 84
85
85 86 def setup(self,path=None,
86 87 startDate=None,
87 88 endDate=None,
@@ -95,7 +96,7 class AMISRReader(ProcessingUnit):
95 96 nBaud = 0,
96 97 online=False):
97 98
98 #print ("T",path)
99
99 100
100 101 self.timezone = timezone
101 102 self.all = all
@@ -126,20 +127,24 class AMISRReader(ProcessingUnit):
126 127 Add code
127 128 '''
128 129 self.isConfig = True
129
130 # print("Setup Done")
130 131 pass
131 132
132 133
133 134 def readAMISRHeader(self,fp):
134 135 header = 'Raw11/Data/RadacHeader'
135 136 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
137 if (self.startDate> datetime.date(2021, 7, 15)): #Se cambió la forma de extracción de Apuntes el 17
136 138 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
137 139 self.trueBeams = self.beamcodeFile.split("\n")
138 140 self.trueBeams.pop()#remove last
139 141 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
140 142 self.beamCode = [int(x, 16) for x in self.realBeamCode]
143 else:
144 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
145 self.beamCode = _beamCode[0,:]
146
141 147
142 #self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
143 148 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
144 149 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
145 150 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
@@ -625,9 +630,6 class AMISRReader(ProcessingUnit):
625 630 diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
626 631 diffUTC = 0
627 632 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
628 #cambio posible 18/02/2020
629
630
631 633
632 634 #print("utc :",indexblock," __ ",t_comp)
633 635 #print(numpy.shape(self.timeset))
@@ -648,7 +650,7 class AMISRReader(ProcessingUnit):
648 650 '''
649 651 This method will be called many times so here you should put all your code
650 652 '''
651
653 #print("running kamisr")
652 654 if not self.isConfig:
653 655 self.setup(**kwargs)
654 656 self.isConfig = True
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
Show file before | Show file after | Hide comments
@@ -174,7 +174,7 def MPDecorator(BaseClass):
174 174 self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
175 175
176 176 self.start_time = time.time()
177 self.err_queue = args[3]
177 #self.err_queue = args[2]
178 178 self.queue = Queue(maxsize=1)
179 179 self.myrun = BaseClass.run
180 180
Show file before | Show file after | Hide comments
@@ -1,4 +1,4
1 #!/usr/bin/env python
1
2 2 import os, sys
3 3 import time
4 4 import datetime
@@ -20,10 +20,10 def main():
20 20 ymax = '300'
21 21 dbmin = '45' #'60'#'55' #'40' #noise esf eej
22 22 dbmax = '65' #'70' #'55'
23 showSPC = '0' #view plot Spectra
24 showRTI = '0' #view plot RTI
25 showNOISE = '0' #view plot NOISE
26 localtime='0' #para ajustar el horario en las gráficas '0' para dejar en utc
23 showSPC = '1' #view plot Spectra
24 showRTI = '1' #view plot RTI
25 showNOISE = '1' #view plot NOISE
26 localtime='1' #para ajustar el horario en las gráficas '0' para dejar en utc
27 27 code = '1,-1,-1,-1,1,1,1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,1,-1,1,1,-1,1'
28 28 nCode = '1'
29 29 nBaud = '28'
@@ -31,10 +31,11 def main():
31 31 today = time.strftime("%Y/%m/%d")
32 32 #startDate=today
33 33 #endDate=today
34 startDate='2019/12/16'
35 endDate='2019/12/16'
34 startDate='2021/07/11'
35 endDate='2021/07/11'
36 36 #inPath= '/home/soporte/dataAMISR_test/'
37 37 inPath= '/home/soporte/dataAMISR/'
38 inPath= '/media/soporte/UARS_4T_D02/AMISR_DATA/2021/'
38 39 #inPath = '/mnt/data_amisr'
39 40 outPath = '/home/soporte/Data/EEJ'
40 41
@@ -64,19 +65,22 def main():
64 65 endDate=endDate, #endDate '2014/10/07',
65 66 startTime='07:01:30',#'07:00:00',
66 67 endTime='19:00:00',#'15:00:00',
67 walk=0,
68 walk=1,
68 69 code = code,
69 70 nCode = nCode,
70 71 nBaud = nBaud,
71 timezone='ut',
72 timezone='lt',
72 73 online=0)
73 74
75
74 76 #AMISR Processing Unit
75 77 ##.......................................................................................
76 78 ##.......................................................................................
77 79 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
78 opObj10 = procUnitConfObj0.addOperation(name='setRadarFrequency')
79 opObj10.addParameter(name='frequency', value='445e6', format='float')
80 opObj10 = procUnitConfObj0.addOperation(name='setAttribute')
81 opObj10.addParameter(name='frequency', value='445.09e6')
82 # opObj10 = procUnitConfObj0.addOperation(name='setRadarFrequency')
83 # opObj10.addParameter(name='frequency', value='445e6', format='float')
80 84
81 85
82 86 opObj01 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
@@ -86,8 +90,8 def main():
86 90 opObj01.addParameter(name='osamp', value=nosamp, format='int')
87 91
88 92
89 opObj02 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
90 opObj02.addParameter(name='n', value='2', format='int')
93 # opObj02 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
94 # opObj02.addParameter(name='n', value='2', format='int')
91 95
92 96
93 97
@@ -106,9 +110,9 def main():
106 110 ##.......................................................................................
107 111 ##.......................................................................................
108 112
109 opObj13 = procUnitConfObj1.addOperation(name='getNoise' , optype ='self')
110 opObj13.addParameter(name='minHei', value='100', format='float')
111 opObj13.addParameter(name='maxHei', value='280', format='float')
113 # opObj13 = procUnitConfObj1.addOperation(name='getNoise' , optype ='self')
114 # opObj13.addParameter(name='minHei', value='100', format='float')
115 # opObj13.addParameter(name='maxHei', value='280', format='float')
112 116
113 117
114 118 #
@@ -138,21 +142,21 def main():
138 142 opObj14.addParameter(name='id', value='3', format='int')
139 143 opObj14.addParameter(name='wintitle', value='title0', format='str')
140 144 opObj14.addParameter(name='showprofile', value='0', format='int')
141 opObj14.addParameter(name='xmin', value=xmin, format='int')
142 opObj14.addParameter(name='xmax', value=xmax, format='int')
145 opObj14.addParameter(name='tmin', value=xmin, format='int')
146 opObj14.addParameter(name='tmax', value=xmax, format='int')
143 147 opObj14.addParameter(name='ymin', value=dbmin, format='int')
144 148 opObj14.addParameter(name='ymax', value=dbmax, format='int')
145 149 opObj14.addParameter(name='save', value=outPath, format='str')
146 150 opObj14.addParameter(name='localtime', value=localtime,format='int')
147 151 opObj14.addParameter(name='show', value = showNOISE, format='int')
148 152
149
153 #
150 154 opObj15 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
151 155 opObj15.addParameter(name='id', value='2', format='int')
152 156 opObj15.addParameter(name='localtime', value=localtime,format='int')
153 157 opObj15.addParameter(name='wintitle', value='RTI', format='str')
154 opObj15.addParameter(name='xmin', value=xmin, format='int')
155 opObj15.addParameter(name='xmax', value=xmax, format='int') #max value =23
158 opObj15.addParameter(name='tmin', value=xmin, format='int')
159 opObj15.addParameter(name='tmax', value=xmax, format='int') #max value =23
156 160 opObj15.addParameter(name='ymin', value=ymin, format='int')
157 161 opObj15.addParameter(name='zmin', value=dbmin, format='int')
158 162 opObj15.addParameter(name='zmax', value=dbmax, format='int')
@@ -170,16 +174,15 def main():
170 174 opObj16 = procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
171 175
172 176
173 #Using ParamWriter::::
177 #Using HDFWriter::::
174 178 ##.......................................................................................
175 179 ##.......................................................................................
176 opObj17 = procUnitConfObj2.addOperation(name='ParamWriter', optype='external')
180 opObj17 = procUnitConfObj2.addOperation(name='HDFWriter', optype='external')
177 181 opObj17.addParameter(name='path', value=outPath)
178 182 opObj17.addParameter(name='blocksPerFile', value='10', format='int')
179 183 opObj17.addParameter(name='metadataList',value='type,inputUnit,heightList',format='list')
180 184 opObj17.addParameter(name='dataList',value='moments,data_SNR,utctime',format='list')
181 opObj17.addParameter(name='mode',value='1',format='int') #'0' channels, '1' parameters, '3' table (for meteors)
182 ##opObj17.addParameter(name='setType', value ='anything', format='str')#no usar
185
183 186
184 187
185 188 ##.......................................................................................
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now