##// END OF EJS Templates
schain
RSS
Separación de datos antes del 17 de Julio del 2021, donde se modificó los apuntes y la lectura de estos
joabAM -
r1371:603b419b2641
parent child
Show More
Side by Side Unified
Context file:
r1371:603b419b2641 Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 <Project description="AMISR EEJ Experiment" id="11" name="Process (eej_proc)"><ReadUnit id="111" inputId="None" name="AMISRReader"><Parameter name="path" value="/media/soporte/UARS_4T_D02/AMISR_DATA/2021/" /><Parameter name="startDate" value="2021/07/11" /><Parameter name="endDate" value="2021/07/11" /><Parameter name="startTime" value="07:01:30" /><Parameter name="endTime" value="19:00:00" /><Parameter name="walk" value="1" /><Parameter name="code" value="(1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1)" /><Parameter name="nCode" value="1" /><Parameter name="nBaud" value="28" /><Parameter name="timezone" value="ut" /><Parameter name="online" value="0" /></ReadUnit><ProcUnit id="112" inputId="111" name="VoltageProc"><Operation id="1121" name="setAttribute"><Parameter name="frequency" value="445090000.0" /></Operation><Operation id="1122" name="Decoder"><Parameter name="code" value="(1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1)" /><Parameter name="nCode" value="1" /><Parameter name="nBaud" value="28" /><Parameter name="osamp" value="1" /></Operation></ProcUnit><ProcUnit id="113" inputId="112" name="SpectraProc"><Parameter name="nFFTPoints" value="16" /><Operation id="1131" name="IncohInt"><Parameter name="n" value="150" /></Operation><Operation id="1132" name="removeDC" /><Operation id="1133" name="SpectraPlot"><Parameter name="id" value="21" /><Parameter name="xaxis" value="velocity" /><Parameter name="ymax" value="300" /><Parameter name="showprofile" value="1" /><Parameter name="wintitle" value="AMISR Beam 0" /><Parameter name="zmin" value="45" /><Parameter name="zmax" value="65" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192/plots" /><Parameter name="colormap" value="jet" /><Parameter name="localtime" value="0" /><Parameter name="show" value="1" /></Operation><Operation id="1134" name="SpectraWriter"><Parameter name="path" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="blocksPerFile" value="10" /></Operation><Operation id="1135" name="NoisePlot"><Parameter name="id" value="3" /><Parameter name="wintitle" value="title0" /><Parameter name="showprofile" value="0" /><Parameter name="xmin" value="07" /><Parameter name="xmax" value="18" /><Parameter name="ymin" value="45" /><Parameter name="ymax" value="65" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="localtime" value="0" /><Parameter name="show" value="0" /></Operation><Operation id="1136" name="RTIPlot"><Parameter name="id" value="2" /><Parameter name="localtime" value="0" /><Parameter name="wintitle" value="RTI" /><Parameter name="xmin" value="07" /><Parameter name="xmax" value="18" /><Parameter name="ymin" value="0" /><Parameter name="zmin" value="45" /><Parameter name="zmax" value="65" /><Parameter name="showprofile" value="0" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192/plots" /><Parameter name="colormap" value="jet" /><Parameter name="show" value="1" /></Operation></ProcUnit><ProcUnit id="114" inputId="113" name="ParametersProc"><Operation id="1141" name="SpectralMoments" /><Operation id="1142" name="ParamWriter"><Parameter name="path" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="blocksPerFile" value="10" /><Parameter name="metadataList" value="['type', 'inputUnit', 'heightList']" /><Parameter name="dataList" value="['moments', 'data_SNR', 'utctime']" /><Parameter name="mode" value="1" /></Operation></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (1577 lines changed) Show them Hide them
@@ -1,918 +1,659
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
2 # All rights reserved.
3 #
3 #
4 # Distributed under the terms of the BSD 3-clause license.
4 # Distributed under the terms of the BSD 3-clause license.
5 """API to create signal chain projects
5 """API to create signal chain projects
6
6
7 The API is provide through class: Project
7 The API is provide through class: Project
8 """
8 """
9
9
10 import re
10 import re
11 import sys
11 import sys
12 import ast
12 import ast
13 import datetime
13 import datetime
14 import traceback
14 import traceback
15 import time
15 import time
16 import multiprocessing
16 import multiprocessing
17 from multiprocessing import Process, Queue
17 from multiprocessing import Process, Queue
18 from threading import Thread
18 from threading import Thread
19 from xml.etree.ElementTree import ElementTree, Element, SubElement
19 from xml.etree.ElementTree import ElementTree, Element, SubElement
20
20
21 from schainpy.admin import Alarm, SchainWarning
21 from schainpy.admin import Alarm, SchainWarning
22 from schainpy.model import *
22 from schainpy.model import *
23 from schainpy.utils import log
23 from schainpy.utils import log
24
24
25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 multiprocessing.set_start_method('fork')
26 multiprocessing.set_start_method('fork')
27
27
28 DTYPES = {
28 class ConfBase():
29 'Voltage': '.r',
29
30 'Spectra': '.pdata'
30 def __init__(self):
31 }
31
32
32 self.id = '0'
33
33 self.name = None
34 def MPProject(project, n=cpu_count()):
34 self.priority = None
35 '''
35 self.parameters = {}
36 Project wrapper to run schain in n processes
36 self.object = None
37 '''
37 self.operations = []
38
38
39 rconf = project.getReadUnitObj()
39 def getId(self):
40 op = rconf.getOperationObj('run')
40
41 dt1 = op.getParameterValue('startDate')
41 return self.id
42 dt2 = op.getParameterValue('endDate')
42
43 tm1 = op.getParameterValue('startTime')
43 def getNewId(self):
44 tm2 = op.getParameterValue('endTime')
44
45 days = (dt2 - dt1).days
45 return int(self.id) * 10 + len(self.operations) + 1
46
46
47 for day in range(days + 1):
47 def updateId(self, new_id):
48 skip = 0
48
49 cursor = 0
49 self.id = str(new_id)
50 processes = []
50
51 dt = dt1 + datetime.timedelta(day)
51 n = 1
52 dt_str = dt.strftime('%Y/%m/%d')
52 for conf in self.operations:
53 reader = JRODataReader()
53 conf_id = str(int(new_id) * 10 + n)
54 paths, files = reader.searchFilesOffLine(path=rconf.path,
54 conf.updateId(conf_id)
55 startDate=dt,
55 n += 1
56 endDate=dt,
56
57 startTime=tm1,
57 def getKwargs(self):
58 endTime=tm2,
58
59 ext=DTYPES[rconf.datatype])
59 params = {}
60 nFiles = len(files)
60
61 if nFiles == 0:
61 for key, value in self.parameters.items():
62 continue
62 if value not in (None, '', ' '):
63 skip = int(math.ceil(nFiles / n))
63 params[key] = value
64 while nFiles > cursor * skip:
64
65 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
65 return params
66 skip=skip)
66
67 p = project.clone()
67 def update(self, **kwargs):
68 p.start()
68
69 processes.append(p)
69 for key, value in kwargs.items():
70 cursor += 1
70 self.addParameter(name=key, value=value)
71
71
72 def beforeExit(exctype, value, trace):
72 def addParameter(self, name, value, format=None):
73 for process in processes:
73 '''
74 process.terminate()
74 '''
75 process.join()
75
76 print(traceback.print_tb(trace))
76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
77
77 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
78 sys.excepthook = beforeExit
78 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
79
79 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
80 for process in processes:
80 else:
81 process.join()
81 try:
82 process.terminate()
82 self.parameters[name] = ast.literal_eval(value)
83
83 except:
84 time.sleep(3)
84 if isinstance(value, str) and ',' in value:
85
85 self.parameters[name] = value.split(',')
86 def wait(context):
86 else:
87
87 self.parameters[name] = value
88 time.sleep(1)
88
89 c = zmq.Context()
89 def getParameters(self):
90 receiver = c.socket(zmq.SUB)
90
91 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
91 params = {}
92 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
92 for key, value in self.parameters.items():
93 msg = receiver.recv_multipart()[1]
93 s = type(value).__name__
94 context.terminate()
94 if s == 'date':
95
95 params[key] = value.strftime('%Y/%m/%d')
96 class ParameterConf():
96 elif s == 'time':
97
97 params[key] = value.strftime('%H:%M:%S')
98 id = None
98 else:
99 name = None
99 params[key] = str(value)
100 value = None
100
101 format = None
101 return params
102
102
103 __formated_value = None
103 def makeXml(self, element):
104
104
105 ELEMENTNAME = 'Parameter'
105 xml = SubElement(element, self.ELEMENTNAME)
106
106 for label in self.xml_labels:
107 def __init__(self):
107 xml.set(label, str(getattr(self, label)))
108
108
109 self.format = 'str'
109 for key, value in self.getParameters().items():
110
110 xml_param = SubElement(xml, 'Parameter')
111 def getElementName(self):
111 xml_param.set('name', key)
112
112 xml_param.set('value', value)
113 return self.ELEMENTNAME
113
114
114 for conf in self.operations:
115 def getValue(self):
115 conf.makeXml(xml)
116
116
117 value = self.value
117 def __str__(self):
118 format = self.format
118
119
119 if self.ELEMENTNAME == 'Operation':
120 if self.__formated_value != None:
120 s = ' {}[id={}]\n'.format(self.name, self.id)
121
121 else:
122 return self.__formated_value
122 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
123
123
124 if format == 'obj':
124 for key, value in self.parameters.items():
125 return value
125 if self.ELEMENTNAME == 'Operation':
126
126 s += ' {}: {}\n'.format(key, value)
127 if format == 'str':
127 else:
128 self.__formated_value = str(value)
128 s += ' {}: {}\n'.format(key, value)
129 return self.__formated_value
129
130
130 for conf in self.operations:
131 if value == '':
131 s += str(conf)
132 raise ValueError('%s: This parameter value is empty' % self.name)
132
133
133 return s
134 if format == 'list':
134
135 strList = [s.strip() for s in value.split(',')]
135 class OperationConf(ConfBase):
136 self.__formated_value = strList
136
137
137 ELEMENTNAME = 'Operation'
138 return self.__formated_value
138 xml_labels = ['id', 'name']
139
139
140 if format == 'intlist':
140 def setup(self, id, name, priority, project_id, err_queue):
141 '''
141
142 Example:
142 self.id = str(id)
143 value = (0,1,2)
143 self.project_id = project_id
144 '''
144 self.name = name
145
145 self.type = 'other'
146 new_value = ast.literal_eval(value)
146 self.err_queue = err_queue
147
147
148 if type(new_value) not in (tuple, list):
148 def readXml(self, element, project_id, err_queue):
149 new_value = [int(new_value)]
149
150
150 self.id = element.get('id')
151 self.__formated_value = new_value
151 self.name = element.get('name')
152
152 self.type = 'other'
153 return self.__formated_value
153 self.project_id = str(project_id)
154
154 self.err_queue = err_queue
155 if format == 'floatlist':
155
156 '''
156 for elm in element.iter('Parameter'):
157 Example:
157 self.addParameter(elm.get('name'), elm.get('value'))
158 value = (0.5, 1.4, 2.7)
158
159 '''
159 def createObject(self):
160
160
161 new_value = ast.literal_eval(value)
161 className = eval(self.name)
162
162
163 if type(new_value) not in (tuple, list):
163 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
164 new_value = [float(new_value)]
164 kwargs = self.getKwargs()
165
165 opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
166 self.__formated_value = new_value
166 opObj.start()
167
167 self.type = 'external'
168 return self.__formated_value
168 else:
169
169 opObj = className()
170 if format == 'date':
170
171 strList = value.split('/')
171 self.object = opObj
172 intList = [int(x) for x in strList]
172 return opObj
173 date = datetime.date(intList[0], intList[1], intList[2])
173
174
174 class ProcUnitConf(ConfBase):
175 self.__formated_value = date
175
176
176 ELEMENTNAME = 'ProcUnit'
177 return self.__formated_value
177 xml_labels = ['id', 'inputId', 'name']
178
178
179 if format == 'time':
179 def setup(self, project_id, id, name, datatype, inputId, err_queue):
180 strList = value.split(':')
180 '''
181 intList = [int(x) for x in strList]
181 '''
182 time = datetime.time(intList[0], intList[1], intList[2])
182
183
183 if datatype == None and name == None:
184 self.__formated_value = time
184 raise ValueError('datatype or name should be defined')
185
185
186 return self.__formated_value
186 if name == None:
187
187 if 'Proc' in datatype:
188 if format == 'pairslist':
188 name = datatype
189 '''
189 else:
190 Example:
190 name = '%sProc' % (datatype)
191 value = (0,1),(1,2)
191
192 '''
192 if datatype == None:
193
193 datatype = name.replace('Proc', '')
194 new_value = ast.literal_eval(value)
194
195
195 self.id = str(id)
196 if type(new_value) not in (tuple, list):
196 self.project_id = project_id
197 raise ValueError('%s has to be a tuple or list of pairs' % value)
197 self.name = name
198
198 self.datatype = datatype
199 if type(new_value[0]) not in (tuple, list):
199 self.inputId = inputId
200 if len(new_value) != 2:
200 self.err_queue = err_queue
201 raise ValueError('%s has to be a tuple or list of pairs' % value)
201 self.operations = []
202 new_value = [new_value]
202 self.parameters = {}
203
203
204 for thisPair in new_value:
204 def removeOperation(self, id):
205 if len(thisPair) != 2:
205
206 raise ValueError('%s has to be a tuple or list of pairs' % value)
206 i = [1 if x.id==id else 0 for x in self.operations]
207
207 self.operations.pop(i.index(1))
208 self.__formated_value = new_value
208
209
209 def getOperation(self, id):
210 return self.__formated_value
210
211
211 for conf in self.operations:
212 if format == 'multilist':
212 if conf.id == id:
213 '''
213 return conf
214 Example:
214
215 value = (0,1,2),(3,4,5)
215 def addOperation(self, name, optype='self'):
216 '''
216 '''
217 multiList = ast.literal_eval(value)
217 '''
218
218
219 if type(multiList[0]) == int:
219 id = self.getNewId()
220 multiList = ast.literal_eval('(' + value + ')')
220 conf = OperationConf()
221
221 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
222 self.__formated_value = multiList
222 self.operations.append(conf)
223
223
224 return self.__formated_value
224 return conf
225
225
226 if format == 'bool':
226 def readXml(self, element, project_id, err_queue):
227 value = int(value)
227
228
228 self.id = element.get('id')
229 if format == 'int':
229 self.name = element.get('name')
230 value = float(value)
230 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
231
231 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
232 format_func = eval(format)
232 self.project_id = str(project_id)
233
233 self.err_queue = err_queue
234 self.__formated_value = format_func(value)
234 self.operations = []
235
235 self.parameters = {}
236 return self.__formated_value
236
237
237 for elm in element:
238 def updateId(self, new_id):
238 if elm.tag == 'Parameter':
239
239 self.addParameter(elm.get('name'), elm.get('value'))
240 self.id = str(new_id)
240 elif elm.tag == 'Operation':
241
241 conf = OperationConf()
242 def setup(self, id, name, value, format='str'):
242 conf.readXml(elm, project_id, err_queue)
243 self.id = str(id)
243 self.operations.append(conf)
244 self.name = name
244
245 if format == 'obj':
245 def createObjects(self):
246 self.value = value
246 '''
247 else:
247 Instancia de unidades de procesamiento.
248 self.value = str(value)
248 '''
249 self.format = str.lower(format)
249
250
250 className = eval(self.name)
251 self.getValue()
251 kwargs = self.getKwargs()
252
252 procUnitObj = className()
253 return 1
253 procUnitObj.name = self.name
254
254 log.success('creating process...', self.name)
255 def update(self, name, value, format='str'):
255
256
256 for conf in self.operations:
257 self.name = name
257
258 self.value = str(value)
258 opObj = conf.createObject()
259 self.format = format
259
260
260 log.success('adding operation: {}, type:{}'.format(
261 def makeXml(self, opElement):
261 conf.name,
262 if self.name not in ('queue',):
262 conf.type), self.name)
263 parmElement = SubElement(opElement, self.ELEMENTNAME)
263
264 parmElement.set('id', str(self.id))
264 procUnitObj.addOperation(conf, opObj)
265 parmElement.set('name', self.name)
265
266 parmElement.set('value', self.value)
266 self.object = procUnitObj
267 parmElement.set('format', self.format)
267
268
268 def run(self):
269 def readXml(self, parmElement):
269 '''
270
270 '''
271 self.id = parmElement.get('id')
271
272 self.name = parmElement.get('name')
272 return self.object.call(**self.getKwargs())
273 self.value = parmElement.get('value')
273
274 self.format = str.lower(parmElement.get('format'))
274
275
275 class ReadUnitConf(ProcUnitConf):
276 # Compatible with old signal chain version
276
277 if self.format == 'int' and self.name == 'idfigure':
277 ELEMENTNAME = 'ReadUnit'
278 self.name = 'id'
278
279
279 def __init__(self):
280 def printattr(self):
280
281
281 self.id = None
282 print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
282 self.datatype = None
283
283 self.name = None
284 class OperationConf():
284 self.inputId = None
285
285 self.operations = []
286 ELEMENTNAME = 'Operation'
286 self.parameters = {}
287
287
288 def __init__(self):
288 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
289
289 startTime='', endTime='', server=None, **kwargs):
290 self.id = '0'
290
291 self.name = None
291 if datatype == None and name == None:
292 self.priority = None
292 raise ValueError('datatype or name should be defined')
293 self.parameters = {}
293 if name == None:
294 self.object = None
294 if 'Reader' in datatype:
295 self.operations = []
295 name = datatype
296
296 datatype = name.replace('Reader','')
297 def getId(self):
297 else:
298
298 name = '{}Reader'.format(datatype)
299 return self.id
299 if datatype == None:
300
300 if 'Reader' in name:
301 def getNewId(self):
301 datatype = name.replace('Reader','')
302
302 else:
303 return int(self.id) * 10 + len(self.operations) + 1
303 datatype = name
304
304 name = '{}Reader'.format(name)
305 def updateId(self, new_id):
305
306
306 self.id = id
307 self.id = str(new_id)
307 self.project_id = project_id
308
308 self.name = name
309 n = 1
309 self.datatype = datatype
310 for conf in self.operations:
310 self.err_queue = err_queue
311 conf_id = str(int(new_id) * 10 + n)
311
312 conf.updateId(conf_id)
312 self.addParameter(name='path', value=path)
313 n += 1
313 self.addParameter(name='startDate', value=startDate)
314
314 self.addParameter(name='endDate', value=endDate)
315 def getKwargs(self):
315 self.addParameter(name='startTime', value=startTime)
316
316 self.addParameter(name='endTime', value=endTime)
317 params = {}
317
318
318 for key, value in kwargs.items():
319 for key, value in self.parameters.items():
319 self.addParameter(name=key, value=value)
320 if value not in (None, '', ' '):
320
321 params[key] = value
321
322
322 class Project(Process):
323 return params
323 """API to create signal chain projects"""
324
324
325 def update(self, **kwargs):
325 ELEMENTNAME = 'Project'
326
326
327 for key, value in kwargs.items():
327 def __init__(self, name=''):
328 self.addParameter(name=key, value=value)
328
329
329 Process.__init__(self)
330 def addParameter(self, name, value, format=None):
330 self.id = '1'
331 '''
331 if name:
332 '''
332 self.name = '{} ({})'.format(Process.__name__, name)
333
333 self.filename = None
334 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
334 self.description = None
335 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
335 self.email = None
336 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
336 self.alarm = []
337 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
337 self.configurations = {}
338 else:
338 # self.err_queue = Queue()
339 try:
339 self.err_queue = None
340 self.parameters[name] = ast.literal_eval(value)
340 self.started = False
341 except:
341
342 if isinstance(value, str) and ',' in value:
342 def getNewId(self):
343 self.parameters[name] = value.split(',')
343
344 else:
344 idList = list(self.configurations.keys())
345 self.parameters[name] = value
345 id = int(self.id) * 10
346
346
347 def getParameters(self):
347 while True:
348
348 id += 1
349 params = {}
349
350 for key, value in self.parameters.items():
350 if str(id) in idList:
351 s = type(value).__name__
351 continue
352 if s == 'date':
352
353 params[key] = value.strftime('%Y/%m/%d')
353 break
354 elif s == 'time':
354
355 params[key] = value.strftime('%H:%M:%S')
355 return str(id)
356 else:
356
357 params[key] = str(value)
357 def updateId(self, new_id):
358
358
359 return params
359 self.id = str(new_id)
360
360
361 def makeXml(self, element):
361 keyList = list(self.configurations.keys())
362
362 keyList.sort()
363 xml = SubElement(element, self.ELEMENTNAME)
363
364 for label in self.xml_labels:
364 n = 1
365 xml.set(label, str(getattr(self, label)))
365 new_confs = {}
366
366
367 for key, value in self.getParameters().items():
367 for procKey in keyList:
368 xml_param = SubElement(xml, 'Parameter')
368
369 xml_param.set('name', key)
369 conf = self.configurations[procKey]
370 xml_param.set('value', value)
370 idProcUnit = str(int(self.id) * 10 + n)
371
371 conf.updateId(idProcUnit)
372 for conf in self.operations:
372 new_confs[idProcUnit] = conf
373 conf.makeXml(xml)
373 n += 1
374
374
375 def __str__(self):
375 self.configurations = new_confs
376
376
377 if self.ELEMENTNAME == 'Operation':
377 def setup(self, id=1, name='', description='', email=None, alarm=[]):
378 s = ' {}[id={}]\n'.format(self.name, self.id)
378
379 else:
379 self.id = str(id)
380 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
380 self.description = description
381
381 self.email = email
382 for key, value in self.parameters.items():
382 self.alarm = alarm
383 if self.ELEMENTNAME == 'Operation':
383 if name:
384 s += ' {}: {}\n'.format(key, value)
384 self.name = '{} ({})'.format(Process.__name__, name)
385 else:
385
386 s += ' {}: {}\n'.format(key, value)
386 def update(self, **kwargs):
387
387
388 for conf in self.operations:
388 for key, value in kwargs.items():
389 s += str(conf)
389 setattr(self, key, value)
390
390
391 return s
391 def clone(self):
392
392
393 class OperationConf(ConfBase):
393 p = Project()
394
394 p.id = self.id
395 ELEMENTNAME = 'Operation'
395 p.name = self.name
396 xml_labels = ['id', 'name']
396 p.description = self.description
397
397 p.configurations = self.configurations.copy()
398 def setup(self, id, name, priority, project_id, err_queue):
398
399
399 return p
400 self.id = str(id)
400
401 self.project_id = project_id
401 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
402 self.name = name
402
403 self.type = 'other'
403 '''
404 self.err_queue = err_queue
404 '''
405
405
406 def readXml(self, element, project_id, err_queue):
406 if id is None:
407
407 idReadUnit = self.getNewId()
408 self.id = element.get('id')
408 else:
409 self.name = element.get('name')
409 idReadUnit = str(id)
410 self.type = 'other'
410
411 self.project_id = str(project_id)
411 conf = ReadUnitConf()
412 self.err_queue = err_queue
412 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
413
413 self.configurations[conf.id] = conf
414 for elm in element.iter('Parameter'):
414
415 self.addParameter(elm.get('name'), elm.get('value'))
415 return conf
416
416
417 def createObject(self):
417 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
418
418
419 className = eval(self.name)
419 '''
420
420 '''
421 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
421
422 kwargs = self.getKwargs()
422 if id is None:
423 opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
423 idProcUnit = self.getNewId()
424 opObj.start()
424 else:
425 self.type = 'external'
425 idProcUnit = id
426 else:
426
427 opObj = className()
427 conf = ProcUnitConf()
428
428 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
429 self.object = opObj
429 self.configurations[conf.id] = conf
430 return opObj
430
431
431 return conf
432 class ProcUnitConf(ConfBase):
432
433
433 def removeProcUnit(self, id):
434 ELEMENTNAME = 'ProcUnit'
434
435 xml_labels = ['id', 'inputId', 'name']
435 if id in self.configurations:
436
436 self.configurations.pop(id)
437 def setup(self, project_id, id, name, datatype, inputId, err_queue):
437
438 '''
438 def getReadUnit(self):
439 '''
439
440
440 for obj in list(self.configurations.values()):
441 if datatype == None and name == None:
441 if obj.ELEMENTNAME == 'ReadUnit':
442 raise ValueError('datatype or name should be defined')
442 return obj
443
443
444 if name == None:
444 return None
445 if 'Proc' in datatype:
445
446 name = datatype
446 def getProcUnit(self, id):
447 else:
447
448 name = '%sProc' % (datatype)
448 return self.configurations[id]
449
449
450 if datatype == None:
450 def getUnits(self):
451 datatype = name.replace('Proc', '')
451
452
452 keys = list(self.configurations)
453 self.id = str(id)
453 keys.sort()
454 self.project_id = project_id
454
455 self.name = name
455 for key in keys:
456 self.datatype = datatype
456 yield self.configurations[key]
457 self.inputId = inputId
457
458 self.err_queue = err_queue
458 def updateUnit(self, id, **kwargs):
459 self.operations = []
459
460 self.parameters = {}
460 conf = self.configurations[id].update(**kwargs)
461
461
462 def removeOperation(self, id):
462 def makeXml(self):
463
463
464 i = [1 if x.id==id else 0 for x in self.operations]
464 xml = Element('Project')
465 self.operations.pop(i.index(1))
465 xml.set('id', str(self.id))
466
466 xml.set('name', self.name)
467 def getOperation(self, id):
467 xml.set('description', self.description)
468
468
469 for conf in self.operations:
469 for conf in self.configurations.values():
470 if conf.id == id:
470 conf.makeXml(xml)
471 return conf
471
472
472 self.xml = xml
473 def addOperation(self, name, optype='self'):
473
474 '''
474 def writeXml(self, filename=None):
475 '''
475
476
476 if filename == None:
477 id = self.getNewId()
477 if self.filename:
478 conf = OperationConf()
478 filename = self.filename
479 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
479 else:
480 self.operations.append(conf)
480 filename = 'schain.xml'
481
481
482 return conf
482 if not filename:
483
483 print('filename has not been defined. Use setFilename(filename) for do it.')
484 def readXml(self, element, project_id, err_queue):
484 return 0
485
485
486 self.id = element.get('id')
486 abs_file = os.path.abspath(filename)
487 self.name = element.get('name')
487
488 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
488 if not os.access(os.path.dirname(abs_file), os.W_OK):
489 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
489 print('No write permission on %s' % os.path.dirname(abs_file))
490 self.project_id = str(project_id)
490 return 0
491 self.err_queue = err_queue
491
492 self.operations = []
492 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
493 self.parameters = {}
493 print('File %s already exists and it could not be overwriten' % abs_file)
494
494 return 0
495 for elm in element:
495
496 if elm.tag == 'Parameter':
496 self.makeXml()
497 self.addParameter(elm.get('name'), elm.get('value'))
497
498 elif elm.tag == 'Operation':
498 ElementTree(self.xml).write(abs_file, method='xml')
499 conf = OperationConf()
499
500 conf.readXml(elm, project_id, err_queue)
500 self.filename = abs_file
501 self.operations.append(conf)
501
502
502 return 1
503 def createObjects(self):
503
504 '''
504 def readXml(self, filename):
505 Instancia de unidades de procesamiento.
505
506 '''
506 abs_file = os.path.abspath(filename)
507
507
508 className = eval(self.name)
508 self.configurations = {}
509 #print(self.name)
509
510 kwargs = self.getKwargs()
510 try:
511 procUnitObj = className()
511 self.xml = ElementTree().parse(abs_file)
512 procUnitObj.name = self.name
512 except:
513 log.success('creating process...', self.name)
513 log.error('Error reading %s, verify file format' % filename)
514
514 return 0
515 for conf in self.operations:
515
516
516 self.id = self.xml.get('id')
517 opObj = conf.createObject()
517 self.name = self.xml.get('name')
518
518 self.description = self.xml.get('description')
519 log.success('adding operation: {}, type:{}'.format(
519
520 conf.name,
520 for element in self.xml:
521 conf.type), self.name)
521 if element.tag == 'ReadUnit':
522
522 conf = ReadUnitConf()
523 procUnitObj.addOperation(conf, opObj)
523 conf.readXml(element, self.id, self.err_queue)
524
524 self.configurations[conf.id] = conf
525 self.object = procUnitObj
525 elif element.tag == 'ProcUnit':
526
526 conf = ProcUnitConf()
527 def run(self):
527 input_proc = self.configurations[element.get('inputId')]
528 '''
528 conf.readXml(element, self.id, self.err_queue)
529 '''
529 self.configurations[conf.id] = conf
530
530
531 return self.object.call(**self.getKwargs())
531 self.filename = abs_file
532
532
533
533 return 1
534 class ReadUnitConf(ProcUnitConf):
534
535
535 def __str__(self):
536 ELEMENTNAME = 'ReadUnit'
536
537
537 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
538 def __init__(self):
538 self.id,
539
539 self.name,
540 self.id = None
540 self.description,
541 self.datatype = None
541 )
542 self.name = None
542
543 self.inputId = None
543 for conf in self.configurations.values():
544 self.operations = []
544 text += '{}'.format(conf)
545 self.parameters = {}
545
546
546 return text
547 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
547
548 startTime='', endTime='', server=None, **kwargs):
548 def createObjects(self):
549
549
550 if datatype == None and name == None:
550 keys = list(self.configurations.keys())
551 raise ValueError('datatype or name should be defined')
551 keys.sort()
552 if name == None:
552 for key in keys:
553 if 'Reader' in datatype:
553 conf = self.configurations[key]
554 name = datatype
554 conf.createObjects()
555 datatype = name.replace('Reader','')
555 if conf.inputId is not None:
556 else:
556 conf.object.setInput(self.configurations[conf.inputId].object)
557 name = '{}Reader'.format(datatype)
557
558 if datatype == None:
558 def monitor(self):
559 if 'Reader' in name:
559
560 datatype = name.replace('Reader','')
560 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
561 else:
561 t.start()
562 datatype = name
562
563 name = '{}Reader'.format(name)
563 def _monitor(self, queue, ctx):
564
564
565 self.id = id
565 import socket
566 self.project_id = project_id
566
567 self.name = name
567 procs = 0
568 self.datatype = datatype
568 err_msg = ''
569 self.err_queue = err_queue
569
570
570 while True:
571 self.addParameter(name='path', value=path)
571 msg = queue.get()
572 self.addParameter(name='startDate', value=startDate)
572 if '#_start_#' in msg:
573 self.addParameter(name='endDate', value=endDate)
573 procs += 1
574 self.addParameter(name='startTime', value=startTime)
574 elif '#_end_#' in msg:
575 self.addParameter(name='endTime', value=endTime)
575 procs -=1
576
576 else:
577 for key, value in kwargs.items():
577 err_msg = msg
578 self.addParameter(name=key, value=value)
578
579
579 if procs == 0 or 'Traceback' in err_msg:
580
580 break
581 class Project(Process):
581 time.sleep(0.1)
582 """API to create signal chain projects"""
582
583
583 if '|' in err_msg:
584 ELEMENTNAME = 'Project'
584 name, err = err_msg.split('|')
585
585 if 'SchainWarning' in err:
586 def __init__(self, name=''):
586 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
587
587 elif 'SchainError' in err:
588 Process.__init__(self)
588 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
589 self.id = '1'
589 else:
590 if name:
590 log.error(err, name)
591 self.name = '{} ({})'.format(Process.__name__, name)
591 else:
592 self.filename = None
592 name, err = self.name, err_msg
593 self.description = None
593
594 self.email = None
594 time.sleep(1)
595 self.alarm = []
595
596 self.configurations = {}
596 ctx.term()
597 # self.err_queue = Queue()
597
598 self.err_queue = None
598 message = ''.join(err)
599 self.started = False
599
600
600 if err_msg:
601 def getNewId(self):
601 subject = 'SChain v%s: Error running %s\n' % (
602
602 schainpy.__version__, self.name)
603 idList = list(self.configurations.keys())
603
604 id = int(self.id) * 10
604 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
605
605 socket.gethostname())
606 while True:
606 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
607 id += 1
607 subtitle += 'Configuration file: %s\n' % self.filename
608
608 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
609 if str(id) in idList:
609
610 continue
610 readUnitConfObj = self.getReadUnit()
611
611 if readUnitConfObj:
612 break
612 subtitle += '\nInput parameters:\n'
613
613 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
614 return str(id)
614 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
615
615 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
616 def updateId(self, new_id):
616 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
617
617 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
618 self.id = str(new_id)
618
619
619 a = Alarm(
620 keyList = list(self.configurations.keys())
620 modes=self.alarm,
621 keyList.sort()
621 email=self.email,
622
622 message=message,
623 n = 1
623 subject=subject,
624 new_confs = {}
624 subtitle=subtitle,
625
625 filename=self.filename
626 for procKey in keyList:
626 )
627
627
628 conf = self.configurations[procKey]
628 a.start()
629 idProcUnit = str(int(self.id) * 10 + n)
629
630 conf.updateId(idProcUnit)
630 def setFilename(self, filename):
631 new_confs[idProcUnit] = conf
631
632 n += 1
632 self.filename = filename
633
633
634 self.configurations = new_confs
634 def runProcs(self):
635
635
636 def setup(self, id=1, name='', description='', email=None, alarm=[]):
636 err = False
637
637 n = len(self.configurations)
638 self.id = str(id)
638
639 self.description = description
639 while not err:
640 self.email = email
640 for conf in self.getUnits():
641 self.alarm = alarm
641 ok = conf.run()
642 if name:
642 if ok == 'Error':
643 self.name = '{} ({})'.format(Process.__name__, name)
643 n -= 1
644
644 continue
645 def update(self, **kwargs):
645 elif not ok:
646
646 break
647 for key, value in kwargs.items():
647 if n == 0:
648 setattr(self, key, value)
648 err = True
649
649
650 def clone(self):
650 def run(self):
651
651
652 p = Project()
652 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
653 p.id = self.id
653 self.started = True
654 p.name = self.name
654 self.start_time = time.time()
655 p.description = self.description
655 self.createObjects()
656 p.configurations = self.configurations.copy()
656 self.runProcs()
657
657 log.success('{} Done (Time: {:4.2f}s)'.format(
658 return p
658 self.name,
659
659 time.time()-self.start_time), '')
660 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
661
662 '''
663 '''
664
665 if id is None:
666 idReadUnit = self.getNewId()
667 else:
668 idReadUnit = str(id)
669
670 conf = ReadUnitConf()
671 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
672 self.configurations[conf.id] = conf
673
674 return conf
675
676 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
677
678 '''
679 '''
680
681 if id is None:
682 idProcUnit = self.getNewId()
683 else:
684 idProcUnit = id
685
686 conf = ProcUnitConf()
687 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
688 self.configurations[conf.id] = conf
689
690 return conf
691
692 def removeProcUnit(self, id):
693
694 if id in self.configurations:
695 self.configurations.pop(id)
696
697 def getReadUnit(self):
698
699 for obj in list(self.configurations.values()):
700 if obj.ELEMENTNAME == 'ReadUnit':
701 return obj
702
703 return None
704
705 def getProcUnit(self, id):
706
707 return self.configurations[id]
708
709 def getUnits(self):
710
711 keys = list(self.configurations)
712 keys.sort()
713
714 for key in keys:
715 yield self.configurations[key]
716
717 def updateUnit(self, id, **kwargs):
718
719 conf = self.configurations[id].update(**kwargs)
720
721 def makeXml(self):
722
723 xml = Element('Project')
724 xml.set('id', str(self.id))
725 xml.set('name', self.name)
726 xml.set('description', self.description)
727
728 for conf in self.configurations.values():
729 conf.makeXml(xml)
730
731 self.xml = xml
732
733 def writeXml(self, filename=None):
734
735 if filename == None:
736 if self.filename:
737 filename = self.filename
738 else:
739 filename = 'schain.xml'
740
741 if not filename:
742 print('filename has not been defined. Use setFilename(filename) for do it.')
743 return 0
744
745 abs_file = os.path.abspath(filename)
746
747 if not os.access(os.path.dirname(abs_file), os.W_OK):
748 print('No write permission on %s' % os.path.dirname(abs_file))
749 return 0
750
751 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
752 print('File %s already exists and it could not be overwriten' % abs_file)
753 return 0
754
755 self.makeXml()
756
757 ElementTree(self.xml).write(abs_file, method='xml')
758
759 self.filename = abs_file
760
761 return 1
762
763 def readXml(self, filename):
764
765 abs_file = os.path.abspath(filename)
766
767 self.configurations = {}
768
769 try:
770 self.xml = ElementTree().parse(abs_file)
771 except:
772 log.error('Error reading %s, verify file format' % filename)
773 return 0
774
775 self.id = self.xml.get('id')
776 self.name = self.xml.get('name')
777 self.description = self.xml.get('description')
778
779 for element in self.xml:
780 if element.tag == 'ReadUnit':
781 conf = ReadUnitConf()
782 conf.readXml(element, self.id, self.err_queue)
783 self.configurations[conf.id] = conf
784 elif element.tag == 'ProcUnit':
785 conf = ProcUnitConf()
786 input_proc = self.configurations[element.get('inputId')]
787 conf.readXml(element, self.id, self.err_queue)
788 self.configurations[conf.id] = conf
789
790 self.filename = abs_file
791
792 return 1
793
794 def __str__(self):
795
796 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
797 self.id,
798 self.name,
799 self.description,
800 )
801
802 for conf in self.configurations.values():
803 text += '{}'.format(conf)
804
805 return text
806
807 def createObjects(self):
808
809 keys = list(self.configurations.keys())
810 keys.sort()
811 for key in keys:
812 conf = self.configurations[key]
813 conf.createObjects()
814 if conf.inputId is not None:
815 conf.object.setInput(self.configurations[conf.inputId].object)
816
817 def monitor(self):
818
819 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
820 t.start()
821
822 def _monitor(self, queue, ctx):
823
824 import socket
825
826 procs = 0
827 err_msg = ''
828
829 while True:
830 msg = queue.get()
831 if '#_start_#' in msg:
832 procs += 1
833 elif '#_end_#' in msg:
834 procs -=1
835 else:
836 err_msg = msg
837
838 if procs == 0 or 'Traceback' in err_msg:
839 break
840 time.sleep(0.1)
841
842 if '|' in err_msg:
843 name, err = err_msg.split('|')
844 if 'SchainWarning' in err:
845 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
846 elif 'SchainError' in err:
847 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
848 else:
849 log.error(err, name)
850 else:
851 name, err = self.name, err_msg
852
853 time.sleep(1)
854
855 ctx.term()
856
857 message = ''.join(err)
858
859 if err_msg:
860 subject = 'SChain v%s: Error running %s\n' % (
861 schainpy.__version__, self.name)
862
863 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
864 socket.gethostname())
865 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
866 subtitle += 'Configuration file: %s\n' % self.filename
867 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
868
869 readUnitConfObj = self.getReadUnit()
870 if readUnitConfObj:
871 subtitle += '\nInput parameters:\n'
872 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
873 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
874 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
875 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
876 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
877
878 a = Alarm(
879 modes=self.alarm,
880 email=self.email,
881 message=message,
882 subject=subject,
883 subtitle=subtitle,
884 filename=self.filename
885 )
886
887 a.start()
888
889 def setFilename(self, filename):
890
891 self.filename = filename
892
893 def runProcs(self):
894
895 err = False
896 n = len(self.configurations)
897
898 while not err:
899 for conf in self.getUnits():
900 ok = conf.run()
901 if ok == 'Error':
902 n -= 1
903 continue
904 elif not ok:
905 break
906 if n == 0:
907 err = True
908
909 def run(self):
910
911 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
912 self.started = True
913 self.start_time = time.time()
914 self.createObjects()
915 self.runProcs()
916 log.success('{} Done (Time: {:4.2f}s)'.format(
917 self.name,
918 time.time()-self.start_time), '')
Show file before | Show file after | Hide comments
@@ -24,7 +24,7 from schainpy.model.data.jrodata import Voltage
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
25 from numpy import imag
25 from numpy import imag
26
26
27 @MPDecorator
27
28 class AMISRReader(ProcessingUnit):
28 class AMISRReader(ProcessingUnit):
29 '''
29 '''
30 classdocs
30 classdocs
@@ -82,6 +82,7 class AMISRReader(ProcessingUnit):
82 self.dataOut = Voltage()
82 self.dataOut = Voltage()
83 self.dataOut.error=False
83 self.dataOut.error=False
84
84
85
85 def setup(self,path=None,
86 def setup(self,path=None,
86 startDate=None,
87 startDate=None,
87 endDate=None,
88 endDate=None,
@@ -95,7 +96,7 class AMISRReader(ProcessingUnit):
95 nBaud = 0,
96 nBaud = 0,
96 online=False):
97 online=False):
97
98
98 #print ("T",path)
99
99
100
100 self.timezone = timezone
101 self.timezone = timezone
101 self.all = all
102 self.all = all
@@ -126,20 +127,24 class AMISRReader(ProcessingUnit):
126 Add code
127 Add code
127 '''
128 '''
128 self.isConfig = True
129 self.isConfig = True
129
130 # print("Setup Done")
130 pass
131 pass
131
132
132
133
133 def readAMISRHeader(self,fp):
134 def readAMISRHeader(self,fp):
134 header = 'Raw11/Data/RadacHeader'
135 header = 'Raw11/Data/RadacHeader'
135 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
136 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
136 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
137 if (self.startDate> datetime.date(2021, 7, 15)): #Se cambió la forma de extracción de Apuntes el 17
137 self.trueBeams = self.beamcodeFile.split("\n")
138 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
138 self.trueBeams.pop()#remove last
139 self.trueBeams = self.beamcodeFile.split("\n")
139 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
140 self.trueBeams.pop()#remove last
140 self.beamCode = [int(x, 16) for x in self.realBeamCode]
141 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
142 self.beamCode = [int(x, 16) for x in self.realBeamCode]
143 else:
144 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
145 self.beamCode = _beamCode[0,:]
146
141
147
142 #self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
143 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
148 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
144 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
149 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
145 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
150 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
@@ -625,9 +630,6 class AMISRReader(ProcessingUnit):
625 diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
630 diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
626 diffUTC = 0
631 diffUTC = 0
627 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
632 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
628 #cambio posible 18/02/2020
629
630
631
633
632 #print("utc :",indexblock," __ ",t_comp)
634 #print("utc :",indexblock," __ ",t_comp)
633 #print(numpy.shape(self.timeset))
635 #print(numpy.shape(self.timeset))
@@ -648,7 +650,7 class AMISRReader(ProcessingUnit):
648 '''
650 '''
649 This method will be called many times so here you should put all your code
651 This method will be called many times so here you should put all your code
650 '''
652 '''
651
653 #print("running kamisr")
652 if not self.isConfig:
654 if not self.isConfig:
653 self.setup(**kwargs)
655 self.setup(**kwargs)
654 self.isConfig = True
656 self.isConfig = True
Show file before | Show file after | Hide comments
@@ -75,7 +75,7 class SpectraReader(JRODataReader, ProcessingUnit):
75
75
76 self.pts2read_SelfSpectra = 0
76 self.pts2read_SelfSpectra = 0
77 self.pts2read_CrossSpectra = 0
77 self.pts2read_CrossSpectra = 0
78 self.pts2read_DCchannels = 0
78 self.pts2read_DCchannels = 0
79 self.ext = ".pdata"
79 self.ext = ".pdata"
80 self.optchar = "P"
80 self.optchar = "P"
81 self.basicHeaderObj = BasicHeader(LOCALTIME)
81 self.basicHeaderObj = BasicHeader(LOCALTIME)
@@ -162,7 +162,7 class SpectraReader(JRODataReader, ProcessingUnit):
162 Exceptions:
162 Exceptions:
163 Si un bloque leido no es un bloque valido
163 Si un bloque leido no es un bloque valido
164 """
164 """
165
165
166 fpointer = self.fp.tell()
166 fpointer = self.fp.tell()
167
167
168 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
168 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
@@ -364,7 +364,7 class SpectraWriter(JRODataWriter, Operation):
364 data.tofile(self.fp)
364 data.tofile(self.fp)
365
365
366 if self.data_cspc is not None:
366 if self.data_cspc is not None:
367
367
368 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
368 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
369 data = numpy.zeros( numpy.shape(cspc), self.dtype )
369 data = numpy.zeros( numpy.shape(cspc), self.dtype )
370 #print 'data.shape', self.shape_cspc_Buffer
370 #print 'data.shape', self.shape_cspc_Buffer
@@ -376,7 +376,7 class SpectraWriter(JRODataWriter, Operation):
376 data.tofile(self.fp)
376 data.tofile(self.fp)
377
377
378 if self.data_dc is not None:
378 if self.data_dc is not None:
379
379
380 dc = self.data_dc
380 dc = self.data_dc
381 data = numpy.zeros( numpy.shape(dc), self.dtype )
381 data = numpy.zeros( numpy.shape(dc), self.dtype )
382 data['real'] = dc.real
382 data['real'] = dc.real
@@ -524,4 +524,4 class SpectraWriter(JRODataWriter, Operation):
524
524
525 self.processingHeaderObj.processFlags = self.getProcessFlags()
525 self.processingHeaderObj.processFlags = self.getProcessFlags()
526
526
527 self.setBasicHeader() No newline at end of file
527 self.setBasicHeader()
Show file before | Show file after | Hide comments
@@ -174,7 +174,7 def MPDecorator(BaseClass):
174 self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
174 self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
175
175
176 self.start_time = time.time()
176 self.start_time = time.time()
177 self.err_queue = args[3]
177 #self.err_queue = args[2]
178 self.queue = Queue(maxsize=1)
178 self.queue = Queue(maxsize=1)
179 self.myrun = BaseClass.run
179 self.myrun = BaseClass.run
180
180
Show file before | Show file after | Hide comments
@@ -1,4 +1,4
1 #!/usr/bin/env python
1
2 import os, sys
2 import os, sys
3 import time
3 import time
4 import datetime
4 import datetime
@@ -20,10 +20,10 def main():
20 ymax = '300'
20 ymax = '300'
21 dbmin = '45' #'60'#'55' #'40' #noise esf eej
21 dbmin = '45' #'60'#'55' #'40' #noise esf eej
22 dbmax = '65' #'70' #'55'
22 dbmax = '65' #'70' #'55'
23 showSPC = '0' #view plot Spectra
23 showSPC = '1' #view plot Spectra
24 showRTI = '0' #view plot RTI
24 showRTI = '1' #view plot RTI
25 showNOISE = '0' #view plot NOISE
25 showNOISE = '1' #view plot NOISE
26 localtime='0' #para ajustar el horario en las gráficas '0' para dejar en utc
26 localtime='1' #para ajustar el horario en las gráficas '0' para dejar en utc
27 code = '1,-1,-1,-1,1,1,1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,1,-1,1,1,-1,1'
27 code = '1,-1,-1,-1,1,1,1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,1,-1,1,1,-1,1'
28 nCode = '1'
28 nCode = '1'
29 nBaud = '28'
29 nBaud = '28'
@@ -31,10 +31,11 def main():
31 today = time.strftime("%Y/%m/%d")
31 today = time.strftime("%Y/%m/%d")
32 #startDate=today
32 #startDate=today
33 #endDate=today
33 #endDate=today
34 startDate='2019/12/16'
34 startDate='2021/07/11'
35 endDate='2019/12/16'
35 endDate='2021/07/11'
36 #inPath= '/home/soporte/dataAMISR_test/'
36 #inPath= '/home/soporte/dataAMISR_test/'
37 inPath= '/home/soporte/dataAMISR/'
37 inPath= '/home/soporte/dataAMISR/'
38 inPath= '/media/soporte/UARS_4T_D02/AMISR_DATA/2021/'
38 #inPath = '/mnt/data_amisr'
39 #inPath = '/mnt/data_amisr'
39 outPath = '/home/soporte/Data/EEJ'
40 outPath = '/home/soporte/Data/EEJ'
40
41
@@ -64,19 +65,22 def main():
64 endDate=endDate, #endDate '2014/10/07',
65 endDate=endDate, #endDate '2014/10/07',
65 startTime='07:01:30',#'07:00:00',
66 startTime='07:01:30',#'07:00:00',
66 endTime='19:00:00',#'15:00:00',
67 endTime='19:00:00',#'15:00:00',
67 walk=0,
68 walk=1,
68 code = code,
69 code = code,
69 nCode = nCode,
70 nCode = nCode,
70 nBaud = nBaud,
71 nBaud = nBaud,
71 timezone='ut',
72 timezone='lt',
72 online=0)
73 online=0)
73
74
75
74 #AMISR Processing Unit
76 #AMISR Processing Unit
75 ##.......................................................................................
77 ##.......................................................................................
76 ##.......................................................................................
78 ##.......................................................................................
77 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
79 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
78 opObj10 = procUnitConfObj0.addOperation(name='setRadarFrequency')
80 opObj10 = procUnitConfObj0.addOperation(name='setAttribute')
79 opObj10.addParameter(name='frequency', value='445e6', format='float')
81 opObj10.addParameter(name='frequency', value='445.09e6')
82 # opObj10 = procUnitConfObj0.addOperation(name='setRadarFrequency')
83 # opObj10.addParameter(name='frequency', value='445e6', format='float')
80
84
81
85
82 opObj01 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
86 opObj01 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
@@ -86,8 +90,8 def main():
86 opObj01.addParameter(name='osamp', value=nosamp, format='int')
90 opObj01.addParameter(name='osamp', value=nosamp, format='int')
87
91
88
92
89 opObj02 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
93 # opObj02 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
90 opObj02.addParameter(name='n', value='2', format='int')
94 # opObj02.addParameter(name='n', value='2', format='int')
91
95
92
96
93
97
@@ -106,9 +110,9 def main():
106 ##.......................................................................................
110 ##.......................................................................................
107 ##.......................................................................................
111 ##.......................................................................................
108
112
109 opObj13 = procUnitConfObj1.addOperation(name='getNoise' , optype ='self')
113 # opObj13 = procUnitConfObj1.addOperation(name='getNoise' , optype ='self')
110 opObj13.addParameter(name='minHei', value='100', format='float')
114 # opObj13.addParameter(name='minHei', value='100', format='float')
111 opObj13.addParameter(name='maxHei', value='280', format='float')
115 # opObj13.addParameter(name='maxHei', value='280', format='float')
112
116
113
117
114 #
118 #
@@ -138,21 +142,21 def main():
138 opObj14.addParameter(name='id', value='3', format='int')
142 opObj14.addParameter(name='id', value='3', format='int')
139 opObj14.addParameter(name='wintitle', value='title0', format='str')
143 opObj14.addParameter(name='wintitle', value='title0', format='str')
140 opObj14.addParameter(name='showprofile', value='0', format='int')
144 opObj14.addParameter(name='showprofile', value='0', format='int')
141 opObj14.addParameter(name='xmin', value=xmin, format='int')
145 opObj14.addParameter(name='tmin', value=xmin, format='int')
142 opObj14.addParameter(name='xmax', value=xmax, format='int')
146 opObj14.addParameter(name='tmax', value=xmax, format='int')
143 opObj14.addParameter(name='ymin', value=dbmin, format='int')
147 opObj14.addParameter(name='ymin', value=dbmin, format='int')
144 opObj14.addParameter(name='ymax', value=dbmax, format='int')
148 opObj14.addParameter(name='ymax', value=dbmax, format='int')
145 opObj14.addParameter(name='save', value=outPath, format='str')
149 opObj14.addParameter(name='save', value=outPath, format='str')
146 opObj14.addParameter(name='localtime', value=localtime,format='int')
150 opObj14.addParameter(name='localtime', value=localtime,format='int')
147 opObj14.addParameter(name='show', value = showNOISE, format='int')
151 opObj14.addParameter(name='show', value = showNOISE, format='int')
148
152
149
153 #
150 opObj15 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
154 opObj15 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
151 opObj15.addParameter(name='id', value='2', format='int')
155 opObj15.addParameter(name='id', value='2', format='int')
152 opObj15.addParameter(name='localtime', value=localtime,format='int')
156 opObj15.addParameter(name='localtime', value=localtime,format='int')
153 opObj15.addParameter(name='wintitle', value='RTI', format='str')
157 opObj15.addParameter(name='wintitle', value='RTI', format='str')
154 opObj15.addParameter(name='xmin', value=xmin, format='int')
158 opObj15.addParameter(name='tmin', value=xmin, format='int')
155 opObj15.addParameter(name='xmax', value=xmax, format='int') #max value =23
159 opObj15.addParameter(name='tmax', value=xmax, format='int') #max value =23
156 opObj15.addParameter(name='ymin', value=ymin, format='int')
160 opObj15.addParameter(name='ymin', value=ymin, format='int')
157 opObj15.addParameter(name='zmin', value=dbmin, format='int')
161 opObj15.addParameter(name='zmin', value=dbmin, format='int')
158 opObj15.addParameter(name='zmax', value=dbmax, format='int')
162 opObj15.addParameter(name='zmax', value=dbmax, format='int')
@@ -170,16 +174,15 def main():
170 opObj16 = procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
174 opObj16 = procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
171
175
172
176
173 #Using ParamWriter::::
177 #Using HDFWriter::::
174 ##.......................................................................................
178 ##.......................................................................................
175 ##.......................................................................................
179 ##.......................................................................................
176 opObj17 = procUnitConfObj2.addOperation(name='ParamWriter', optype='external')
180 opObj17 = procUnitConfObj2.addOperation(name='HDFWriter', optype='external')
177 opObj17.addParameter(name='path', value=outPath)
181 opObj17.addParameter(name='path', value=outPath)
178 opObj17.addParameter(name='blocksPerFile', value='10', format='int')
182 opObj17.addParameter(name='blocksPerFile', value='10', format='int')
179 opObj17.addParameter(name='metadataList',value='type,inputUnit,heightList',format='list')
183 opObj17.addParameter(name='metadataList',value='type,inputUnit,heightList',format='list')
180 opObj17.addParameter(name='dataList',value='moments,data_SNR,utctime',format='list')
184 opObj17.addParameter(name='dataList',value='moments,data_SNR,utctime',format='list')
181 opObj17.addParameter(name='mode',value='1',format='int') #'0' channels, '1' parameters, '3' table (for meteors)
185
182 ##opObj17.addParameter(name='setType', value ='anything', format='str')#no usar
183
186
184
187
185 ##.......................................................................................
188 ##.......................................................................................
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now