##// END OF EJS Templates
schain
RSS
Separación de datos antes del 17 de Julio del 2021, donde se modificó los apuntes y la lectura de estos
joabAM -
r1371:603b419b2641
parent child
Show More
Side by Side Unified
Context file:
r1371:603b419b2641 Loading diff...:
Show file before | Show file after | Hide comments
@@ -0,0 +1,1
1 <Project description="AMISR EEJ Experiment" id="11" name="Process (eej_proc)"><ReadUnit id="111" inputId="None" name="AMISRReader"><Parameter name="path" value="/media/soporte/UARS_4T_D02/AMISR_DATA/2021/" /><Parameter name="startDate" value="2021/07/11" /><Parameter name="endDate" value="2021/07/11" /><Parameter name="startTime" value="07:01:30" /><Parameter name="endTime" value="19:00:00" /><Parameter name="walk" value="1" /><Parameter name="code" value="(1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1)" /><Parameter name="nCode" value="1" /><Parameter name="nBaud" value="28" /><Parameter name="timezone" value="ut" /><Parameter name="online" value="0" /></ReadUnit><ProcUnit id="112" inputId="111" name="VoltageProc"><Operation id="1121" name="setAttribute"><Parameter name="frequency" value="445090000.0" /></Operation><Operation id="1122" name="Decoder"><Parameter name="code" value="(1, -1, -1, -1, 1, 1, 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1)" /><Parameter name="nCode" value="1" /><Parameter name="nBaud" value="28" /><Parameter name="osamp" value="1" /></Operation></ProcUnit><ProcUnit id="113" inputId="112" name="SpectraProc"><Parameter name="nFFTPoints" value="16" /><Operation id="1131" name="IncohInt"><Parameter name="n" value="150" /></Operation><Operation id="1132" name="removeDC" /><Operation id="1133" name="SpectraPlot"><Parameter name="id" value="21" /><Parameter name="xaxis" value="velocity" /><Parameter name="ymax" value="300" /><Parameter name="showprofile" value="1" /><Parameter name="wintitle" value="AMISR Beam 0" /><Parameter name="zmin" value="45" /><Parameter name="zmax" value="65" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192/plots" /><Parameter name="colormap" value="jet" /><Parameter name="localtime" value="0" /><Parameter name="show" value="1" /></Operation><Operation id="1134" name="SpectraWriter"><Parameter name="path" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="blocksPerFile" value="10" /></Operation><Operation id="1135" name="NoisePlot"><Parameter name="id" value="3" /><Parameter name="wintitle" value="title0" /><Parameter name="showprofile" value="0" /><Parameter name="xmin" value="07" /><Parameter name="xmax" value="18" /><Parameter name="ymin" value="45" /><Parameter name="ymax" value="65" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="localtime" value="0" /><Parameter name="show" value="0" /></Operation><Operation id="1136" name="RTIPlot"><Parameter name="id" value="2" /><Parameter name="localtime" value="0" /><Parameter name="wintitle" value="RTI" /><Parameter name="xmin" value="07" /><Parameter name="xmax" value="18" /><Parameter name="ymin" value="0" /><Parameter name="zmin" value="45" /><Parameter name="zmax" value="65" /><Parameter name="showprofile" value="0" /><Parameter name="save" value="/home/soporte/Data/EEJ/EEJ2021192/plots" /><Parameter name="colormap" value="jet" /><Parameter name="show" value="1" /></Operation></ProcUnit><ProcUnit id="114" inputId="113" name="ParametersProc"><Operation id="1141" name="SpectralMoments" /><Operation id="1142" name="ParamWriter"><Parameter name="path" value="/home/soporte/Data/EEJ/EEJ2021192" /><Parameter name="blocksPerFile" value="10" /><Parameter name="metadataList" value="['type', 'inputUnit', 'heightList']" /><Parameter name="dataList" value="['moments', 'data_SNR', 'utctime']" /><Parameter name="mode" value="1" /></Operation></ProcUnit></Project> No newline at end of file
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (1577 lines changed) Show them Hide them
@@ -1,918 +1,659
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
3 #
4 # Distributed under the terms of the BSD 3-clause license.
5 """API to create signal chain projects
6
7 The API is provide through class: Project
8 """
9
10 import re
11 import sys
12 import ast
13 import datetime
14 import traceback
15 import time
16 import multiprocessing
17 from multiprocessing import Process, Queue
18 from threading import Thread
19 from xml.etree.ElementTree import ElementTree, Element, SubElement
20
21 from schainpy.admin import Alarm, SchainWarning
22 from schainpy.model import *
23 from schainpy.utils import log
24
25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 multiprocessing.set_start_method('fork')
27
28 DTYPES = {
29 'Voltage': '.r',
30 'Spectra': '.pdata'
31 }
32
33
34 def MPProject(project, n=cpu_count()):
35 '''
36 Project wrapper to run schain in n processes
37 '''
38
39 rconf = project.getReadUnitObj()
40 op = rconf.getOperationObj('run')
41 dt1 = op.getParameterValue('startDate')
42 dt2 = op.getParameterValue('endDate')
43 tm1 = op.getParameterValue('startTime')
44 tm2 = op.getParameterValue('endTime')
45 days = (dt2 - dt1).days
46
47 for day in range(days + 1):
48 skip = 0
49 cursor = 0
50 processes = []
51 dt = dt1 + datetime.timedelta(day)
52 dt_str = dt.strftime('%Y/%m/%d')
53 reader = JRODataReader()
54 paths, files = reader.searchFilesOffLine(path=rconf.path,
55 startDate=dt,
56 endDate=dt,
57 startTime=tm1,
58 endTime=tm2,
59 ext=DTYPES[rconf.datatype])
60 nFiles = len(files)
61 if nFiles == 0:
62 continue
63 skip = int(math.ceil(nFiles / n))
64 while nFiles > cursor * skip:
65 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
66 skip=skip)
67 p = project.clone()
68 p.start()
69 processes.append(p)
70 cursor += 1
71
72 def beforeExit(exctype, value, trace):
73 for process in processes:
74 process.terminate()
75 process.join()
76 print(traceback.print_tb(trace))
77
78 sys.excepthook = beforeExit
79
80 for process in processes:
81 process.join()
82 process.terminate()
83
84 time.sleep(3)
85
86 def wait(context):
87
88 time.sleep(1)
89 c = zmq.Context()
90 receiver = c.socket(zmq.SUB)
91 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
92 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
93 msg = receiver.recv_multipart()[1]
94 context.terminate()
95
96 class ParameterConf():
97
98 id = None
99 name = None
100 value = None
101 format = None
102
103 __formated_value = None
104
105 ELEMENTNAME = 'Parameter'
106
107 def __init__(self):
108
109 self.format = 'str'
110
111 def getElementName(self):
112
113 return self.ELEMENTNAME
114
115 def getValue(self):
116
117 value = self.value
118 format = self.format
119
120 if self.__formated_value != None:
121
122 return self.__formated_value
123
124 if format == 'obj':
125 return value
126
127 if format == 'str':
128 self.__formated_value = str(value)
129 return self.__formated_value
130
131 if value == '':
132 raise ValueError('%s: This parameter value is empty' % self.name)
133
134 if format == 'list':
135 strList = [s.strip() for s in value.split(',')]
136 self.__formated_value = strList
137
138 return self.__formated_value
139
140 if format == 'intlist':
141 '''
142 Example:
143 value = (0,1,2)
144 '''
145
146 new_value = ast.literal_eval(value)
147
148 if type(new_value) not in (tuple, list):
149 new_value = [int(new_value)]
150
151 self.__formated_value = new_value
152
153 return self.__formated_value
154
155 if format == 'floatlist':
156 '''
157 Example:
158 value = (0.5, 1.4, 2.7)
159 '''
160
161 new_value = ast.literal_eval(value)
162
163 if type(new_value) not in (tuple, list):
164 new_value = [float(new_value)]
165
166 self.__formated_value = new_value
167
168 return self.__formated_value
169
170 if format == 'date':
171 strList = value.split('/')
172 intList = [int(x) for x in strList]
173 date = datetime.date(intList[0], intList[1], intList[2])
174
175 self.__formated_value = date
176
177 return self.__formated_value
178
179 if format == 'time':
180 strList = value.split(':')
181 intList = [int(x) for x in strList]
182 time = datetime.time(intList[0], intList[1], intList[2])
183
184 self.__formated_value = time
185
186 return self.__formated_value
187
188 if format == 'pairslist':
189 '''
190 Example:
191 value = (0,1),(1,2)
192 '''
193
194 new_value = ast.literal_eval(value)
195
196 if type(new_value) not in (tuple, list):
197 raise ValueError('%s has to be a tuple or list of pairs' % value)
198
199 if type(new_value[0]) not in (tuple, list):
200 if len(new_value) != 2:
201 raise ValueError('%s has to be a tuple or list of pairs' % value)
202 new_value = [new_value]
203
204 for thisPair in new_value:
205 if len(thisPair) != 2:
206 raise ValueError('%s has to be a tuple or list of pairs' % value)
207
208 self.__formated_value = new_value
209
210 return self.__formated_value
211
212 if format == 'multilist':
213 '''
214 Example:
215 value = (0,1,2),(3,4,5)
216 '''
217 multiList = ast.literal_eval(value)
218
219 if type(multiList[0]) == int:
220 multiList = ast.literal_eval('(' + value + ')')
221
222 self.__formated_value = multiList
223
224 return self.__formated_value
225
226 if format == 'bool':
227 value = int(value)
228
229 if format == 'int':
230 value = float(value)
231
232 format_func = eval(format)
233
234 self.__formated_value = format_func(value)
235
236 return self.__formated_value
237
238 def updateId(self, new_id):
239
240 self.id = str(new_id)
241
242 def setup(self, id, name, value, format='str'):
243 self.id = str(id)
244 self.name = name
245 if format == 'obj':
246 self.value = value
247 else:
248 self.value = str(value)
249 self.format = str.lower(format)
250
251 self.getValue()
252
253 return 1
254
255 def update(self, name, value, format='str'):
256
257 self.name = name
258 self.value = str(value)
259 self.format = format
260
261 def makeXml(self, opElement):
262 if self.name not in ('queue',):
263 parmElement = SubElement(opElement, self.ELEMENTNAME)
264 parmElement.set('id', str(self.id))
265 parmElement.set('name', self.name)
266 parmElement.set('value', self.value)
267 parmElement.set('format', self.format)
268
269 def readXml(self, parmElement):
270
271 self.id = parmElement.get('id')
272 self.name = parmElement.get('name')
273 self.value = parmElement.get('value')
274 self.format = str.lower(parmElement.get('format'))
275
276 # Compatible with old signal chain version
277 if self.format == 'int' and self.name == 'idfigure':
278 self.name = 'id'
279
280 def printattr(self):
281
282 print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
283
284 class OperationConf():
285
286 ELEMENTNAME = 'Operation'
287
288 def __init__(self):
289
290 self.id = '0'
291 self.name = None
292 self.priority = None
293 self.parameters = {}
294 self.object = None
295 self.operations = []
296
297 def getId(self):
298
299 return self.id
300
301 def getNewId(self):
302
303 return int(self.id) * 10 + len(self.operations) + 1
304
305 def updateId(self, new_id):
306
307 self.id = str(new_id)
308
309 n = 1
310 for conf in self.operations:
311 conf_id = str(int(new_id) * 10 + n)
312 conf.updateId(conf_id)
313 n += 1
314
315 def getKwargs(self):
316
317 params = {}
318
319 for key, value in self.parameters.items():
320 if value not in (None, '', ' '):
321 params[key] = value
322
323 return params
324
325 def update(self, **kwargs):
326
327 for key, value in kwargs.items():
328 self.addParameter(name=key, value=value)
329
330 def addParameter(self, name, value, format=None):
331 '''
332 '''
333
334 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
335 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
336 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
337 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
338 else:
339 try:
340 self.parameters[name] = ast.literal_eval(value)
341 except:
342 if isinstance(value, str) and ',' in value:
343 self.parameters[name] = value.split(',')
344 else:
345 self.parameters[name] = value
346
347 def getParameters(self):
348
349 params = {}
350 for key, value in self.parameters.items():
351 s = type(value).__name__
352 if s == 'date':
353 params[key] = value.strftime('%Y/%m/%d')
354 elif s == 'time':
355 params[key] = value.strftime('%H:%M:%S')
356 else:
357 params[key] = str(value)
358
359 return params
360
361 def makeXml(self, element):
362
363 xml = SubElement(element, self.ELEMENTNAME)
364 for label in self.xml_labels:
365 xml.set(label, str(getattr(self, label)))
366
367 for key, value in self.getParameters().items():
368 xml_param = SubElement(xml, 'Parameter')
369 xml_param.set('name', key)
370 xml_param.set('value', value)
371
372 for conf in self.operations:
373 conf.makeXml(xml)
374
375 def __str__(self):
376
377 if self.ELEMENTNAME == 'Operation':
378 s = ' {}[id={}]\n'.format(self.name, self.id)
379 else:
380 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
381
382 for key, value in self.parameters.items():
383 if self.ELEMENTNAME == 'Operation':
384 s += ' {}: {}\n'.format(key, value)
385 else:
386 s += ' {}: {}\n'.format(key, value)
387
388 for conf in self.operations:
389 s += str(conf)
390
391 return s
392
393 class OperationConf(ConfBase):
394
395 ELEMENTNAME = 'Operation'
396 xml_labels = ['id', 'name']
397
398 def setup(self, id, name, priority, project_id, err_queue):
399
400 self.id = str(id)
401 self.project_id = project_id
402 self.name = name
403 self.type = 'other'
404 self.err_queue = err_queue
405
406 def readXml(self, element, project_id, err_queue):
407
408 self.id = element.get('id')
409 self.name = element.get('name')
410 self.type = 'other'
411 self.project_id = str(project_id)
412 self.err_queue = err_queue
413
414 for elm in element.iter('Parameter'):
415 self.addParameter(elm.get('name'), elm.get('value'))
416
417 def createObject(self):
418
419 className = eval(self.name)
420
421 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
422 kwargs = self.getKwargs()
423 opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
424 opObj.start()
425 self.type = 'external'
426 else:
427 opObj = className()
428
429 self.object = opObj
430 return opObj
431
432 class ProcUnitConf(ConfBase):
433
434 ELEMENTNAME = 'ProcUnit'
435 xml_labels = ['id', 'inputId', 'name']
436
437 def setup(self, project_id, id, name, datatype, inputId, err_queue):
438 '''
439 '''
440
441 if datatype == None and name == None:
442 raise ValueError('datatype or name should be defined')
443
444 if name == None:
445 if 'Proc' in datatype:
446 name = datatype
447 else:
448 name = '%sProc' % (datatype)
449
450 if datatype == None:
451 datatype = name.replace('Proc', '')
452
453 self.id = str(id)
454 self.project_id = project_id
455 self.name = name
456 self.datatype = datatype
457 self.inputId = inputId
458 self.err_queue = err_queue
459 self.operations = []
460 self.parameters = {}
461
462 def removeOperation(self, id):
463
464 i = [1 if x.id==id else 0 for x in self.operations]
465 self.operations.pop(i.index(1))
466
467 def getOperation(self, id):
468
469 for conf in self.operations:
470 if conf.id == id:
471 return conf
472
473 def addOperation(self, name, optype='self'):
474 '''
475 '''
476
477 id = self.getNewId()
478 conf = OperationConf()
479 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
480 self.operations.append(conf)
481
482 return conf
483
484 def readXml(self, element, project_id, err_queue):
485
486 self.id = element.get('id')
487 self.name = element.get('name')
488 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
489 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
490 self.project_id = str(project_id)
491 self.err_queue = err_queue
492 self.operations = []
493 self.parameters = {}
494
495 for elm in element:
496 if elm.tag == 'Parameter':
497 self.addParameter(elm.get('name'), elm.get('value'))
498 elif elm.tag == 'Operation':
499 conf = OperationConf()
500 conf.readXml(elm, project_id, err_queue)
501 self.operations.append(conf)
502
503 def createObjects(self):
504 '''
505 Instancia de unidades de procesamiento.
506 '''
507
508 className = eval(self.name)
509 #print(self.name)
510 kwargs = self.getKwargs()
511 procUnitObj = className()
512 procUnitObj.name = self.name
513 log.success('creating process...', self.name)
514
515 for conf in self.operations:
516
517 opObj = conf.createObject()
518
519 log.success('adding operation: {}, type:{}'.format(
520 conf.name,
521 conf.type), self.name)
522
523 procUnitObj.addOperation(conf, opObj)
524
525 self.object = procUnitObj
526
527 def run(self):
528 '''
529 '''
530
531 return self.object.call(**self.getKwargs())
532
533
534 class ReadUnitConf(ProcUnitConf):
535
536 ELEMENTNAME = 'ReadUnit'
537
538 def __init__(self):
539
540 self.id = None
541 self.datatype = None
542 self.name = None
543 self.inputId = None
544 self.operations = []
545 self.parameters = {}
546
547 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
548 startTime='', endTime='', server=None, **kwargs):
549
550 if datatype == None and name == None:
551 raise ValueError('datatype or name should be defined')
552 if name == None:
553 if 'Reader' in datatype:
554 name = datatype
555 datatype = name.replace('Reader','')
556 else:
557 name = '{}Reader'.format(datatype)
558 if datatype == None:
559 if 'Reader' in name:
560 datatype = name.replace('Reader','')
561 else:
562 datatype = name
563 name = '{}Reader'.format(name)
564
565 self.id = id
566 self.project_id = project_id
567 self.name = name
568 self.datatype = datatype
569 self.err_queue = err_queue
570
571 self.addParameter(name='path', value=path)
572 self.addParameter(name='startDate', value=startDate)
573 self.addParameter(name='endDate', value=endDate)
574 self.addParameter(name='startTime', value=startTime)
575 self.addParameter(name='endTime', value=endTime)
576
577 for key, value in kwargs.items():
578 self.addParameter(name=key, value=value)
579
580
581 class Project(Process):
582 """API to create signal chain projects"""
583
584 ELEMENTNAME = 'Project'
585
586 def __init__(self, name=''):
587
588 Process.__init__(self)
589 self.id = '1'
590 if name:
591 self.name = '{} ({})'.format(Process.__name__, name)
592 self.filename = None
593 self.description = None
594 self.email = None
595 self.alarm = []
596 self.configurations = {}
597 # self.err_queue = Queue()
598 self.err_queue = None
599 self.started = False
600
601 def getNewId(self):
602
603 idList = list(self.configurations.keys())
604 id = int(self.id) * 10
605
606 while True:
607 id += 1
608
609 if str(id) in idList:
610 continue
611
612 break
613
614 return str(id)
615
616 def updateId(self, new_id):
617
618 self.id = str(new_id)
619
620 keyList = list(self.configurations.keys())
621 keyList.sort()
622
623 n = 1
624 new_confs = {}
625
626 for procKey in keyList:
627
628 conf = self.configurations[procKey]
629 idProcUnit = str(int(self.id) * 10 + n)
630 conf.updateId(idProcUnit)
631 new_confs[idProcUnit] = conf
632 n += 1
633
634 self.configurations = new_confs
635
636 def setup(self, id=1, name='', description='', email=None, alarm=[]):
637
638 self.id = str(id)
639 self.description = description
640 self.email = email
641 self.alarm = alarm
642 if name:
643 self.name = '{} ({})'.format(Process.__name__, name)
644
645 def update(self, **kwargs):
646
647 for key, value in kwargs.items():
648 setattr(self, key, value)
649
650 def clone(self):
651
652 p = Project()
653 p.id = self.id
654 p.name = self.name
655 p.description = self.description
656 p.configurations = self.configurations.copy()
657
658 return p
659
660 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
661
662 '''
663 '''
664
665 if id is None:
666 idReadUnit = self.getNewId()
667 else:
668 idReadUnit = str(id)
669
670 conf = ReadUnitConf()
671 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
672 self.configurations[conf.id] = conf
673
674 return conf
675
676 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
677
678 '''
679 '''
680
681 if id is None:
682 idProcUnit = self.getNewId()
683 else:
684 idProcUnit = id
685
686 conf = ProcUnitConf()
687 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
688 self.configurations[conf.id] = conf
689
690 return conf
691
692 def removeProcUnit(self, id):
693
694 if id in self.configurations:
695 self.configurations.pop(id)
696
697 def getReadUnit(self):
698
699 for obj in list(self.configurations.values()):
700 if obj.ELEMENTNAME == 'ReadUnit':
701 return obj
702
703 return None
704
705 def getProcUnit(self, id):
706
707 return self.configurations[id]
708
709 def getUnits(self):
710
711 keys = list(self.configurations)
712 keys.sort()
713
714 for key in keys:
715 yield self.configurations[key]
716
717 def updateUnit(self, id, **kwargs):
718
719 conf = self.configurations[id].update(**kwargs)
720
721 def makeXml(self):
722
723 xml = Element('Project')
724 xml.set('id', str(self.id))
725 xml.set('name', self.name)
726 xml.set('description', self.description)
727
728 for conf in self.configurations.values():
729 conf.makeXml(xml)
730
731 self.xml = xml
732
733 def writeXml(self, filename=None):
734
735 if filename == None:
736 if self.filename:
737 filename = self.filename
738 else:
739 filename = 'schain.xml'
740
741 if not filename:
742 print('filename has not been defined. Use setFilename(filename) for do it.')
743 return 0
744
745 abs_file = os.path.abspath(filename)
746
747 if not os.access(os.path.dirname(abs_file), os.W_OK):
748 print('No write permission on %s' % os.path.dirname(abs_file))
749 return 0
750
751 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
752 print('File %s already exists and it could not be overwriten' % abs_file)
753 return 0
754
755 self.makeXml()
756
757 ElementTree(self.xml).write(abs_file, method='xml')
758
759 self.filename = abs_file
760
761 return 1
762
763 def readXml(self, filename):
764
765 abs_file = os.path.abspath(filename)
766
767 self.configurations = {}
768
769 try:
770 self.xml = ElementTree().parse(abs_file)
771 except:
772 log.error('Error reading %s, verify file format' % filename)
773 return 0
774
775 self.id = self.xml.get('id')
776 self.name = self.xml.get('name')
777 self.description = self.xml.get('description')
778
779 for element in self.xml:
780 if element.tag == 'ReadUnit':
781 conf = ReadUnitConf()
782 conf.readXml(element, self.id, self.err_queue)
783 self.configurations[conf.id] = conf
784 elif element.tag == 'ProcUnit':
785 conf = ProcUnitConf()
786 input_proc = self.configurations[element.get('inputId')]
787 conf.readXml(element, self.id, self.err_queue)
788 self.configurations[conf.id] = conf
789
790 self.filename = abs_file
791
792 return 1
793
794 def __str__(self):
795
796 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
797 self.id,
798 self.name,
799 self.description,
800 )
801
802 for conf in self.configurations.values():
803 text += '{}'.format(conf)
804
805 return text
806
807 def createObjects(self):
808
809 keys = list(self.configurations.keys())
810 keys.sort()
811 for key in keys:
812 conf = self.configurations[key]
813 conf.createObjects()
814 if conf.inputId is not None:
815 conf.object.setInput(self.configurations[conf.inputId].object)
816
817 def monitor(self):
818
819 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
820 t.start()
821
822 def _monitor(self, queue, ctx):
823
824 import socket
825
826 procs = 0
827 err_msg = ''
828
829 while True:
830 msg = queue.get()
831 if '#_start_#' in msg:
832 procs += 1
833 elif '#_end_#' in msg:
834 procs -=1
835 else:
836 err_msg = msg
837
838 if procs == 0 or 'Traceback' in err_msg:
839 break
840 time.sleep(0.1)
841
842 if '|' in err_msg:
843 name, err = err_msg.split('|')
844 if 'SchainWarning' in err:
845 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
846 elif 'SchainError' in err:
847 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
848 else:
849 log.error(err, name)
850 else:
851 name, err = self.name, err_msg
852
853 time.sleep(1)
854
855 ctx.term()
856
857 message = ''.join(err)
858
859 if err_msg:
860 subject = 'SChain v%s: Error running %s\n' % (
861 schainpy.__version__, self.name)
862
863 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
864 socket.gethostname())
865 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
866 subtitle += 'Configuration file: %s\n' % self.filename
867 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
868
869 readUnitConfObj = self.getReadUnit()
870 if readUnitConfObj:
871 subtitle += '\nInput parameters:\n'
872 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
873 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
874 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
875 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
876 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
877
878 a = Alarm(
879 modes=self.alarm,
880 email=self.email,
881 message=message,
882 subject=subject,
883 subtitle=subtitle,
884 filename=self.filename
885 )
886
887 a.start()
888
889 def setFilename(self, filename):
890
891 self.filename = filename
892
893 def runProcs(self):
894
895 err = False
896 n = len(self.configurations)
897
898 while not err:
899 for conf in self.getUnits():
900 ok = conf.run()
901 if ok == 'Error':
902 n -= 1
903 continue
904 elif not ok:
905 break
906 if n == 0:
907 err = True
908
909 def run(self):
910
911 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
912 self.started = True
913 self.start_time = time.time()
914 self.createObjects()
915 self.runProcs()
916 log.success('{} Done (Time: {:4.2f}s)'.format(
917 self.name,
918 time.time()-self.start_time), '')
1 # Copyright (c) 2012-2020 Jicamarca Radio Observatory
2 # All rights reserved.
3 #
4 # Distributed under the terms of the BSD 3-clause license.
5 """API to create signal chain projects
6
7 The API is provide through class: Project
8 """
9
10 import re
11 import sys
12 import ast
13 import datetime
14 import traceback
15 import time
16 import multiprocessing
17 from multiprocessing import Process, Queue
18 from threading import Thread
19 from xml.etree.ElementTree import ElementTree, Element, SubElement
20
21 from schainpy.admin import Alarm, SchainWarning
22 from schainpy.model import *
23 from schainpy.utils import log
24
25 if 'darwin' in sys.platform and sys.version_info[0] == 3 and sys.version_info[1] > 7:
26 multiprocessing.set_start_method('fork')
27
28 class ConfBase():
29
30 def __init__(self):
31
32 self.id = '0'
33 self.name = None
34 self.priority = None
35 self.parameters = {}
36 self.object = None
37 self.operations = []
38
39 def getId(self):
40
41 return self.id
42
43 def getNewId(self):
44
45 return int(self.id) * 10 + len(self.operations) + 1
46
47 def updateId(self, new_id):
48
49 self.id = str(new_id)
50
51 n = 1
52 for conf in self.operations:
53 conf_id = str(int(new_id) * 10 + n)
54 conf.updateId(conf_id)
55 n += 1
56
57 def getKwargs(self):
58
59 params = {}
60
61 for key, value in self.parameters.items():
62 if value not in (None, '', ' '):
63 params[key] = value
64
65 return params
66
67 def update(self, **kwargs):
68
69 for key, value in kwargs.items():
70 self.addParameter(name=key, value=value)
71
72 def addParameter(self, name, value, format=None):
73 '''
74 '''
75
76 if isinstance(value, str) and re.search(r'(\d+/\d+/\d+)', value):
77 self.parameters[name] = datetime.date(*[int(x) for x in value.split('/')])
78 elif isinstance(value, str) and re.search(r'(\d+:\d+:\d+)', value):
79 self.parameters[name] = datetime.time(*[int(x) for x in value.split(':')])
80 else:
81 try:
82 self.parameters[name] = ast.literal_eval(value)
83 except:
84 if isinstance(value, str) and ',' in value:
85 self.parameters[name] = value.split(',')
86 else:
87 self.parameters[name] = value
88
89 def getParameters(self):
90
91 params = {}
92 for key, value in self.parameters.items():
93 s = type(value).__name__
94 if s == 'date':
95 params[key] = value.strftime('%Y/%m/%d')
96 elif s == 'time':
97 params[key] = value.strftime('%H:%M:%S')
98 else:
99 params[key] = str(value)
100
101 return params
102
103 def makeXml(self, element):
104
105 xml = SubElement(element, self.ELEMENTNAME)
106 for label in self.xml_labels:
107 xml.set(label, str(getattr(self, label)))
108
109 for key, value in self.getParameters().items():
110 xml_param = SubElement(xml, 'Parameter')
111 xml_param.set('name', key)
112 xml_param.set('value', value)
113
114 for conf in self.operations:
115 conf.makeXml(xml)
116
117 def __str__(self):
118
119 if self.ELEMENTNAME == 'Operation':
120 s = ' {}[id={}]\n'.format(self.name, self.id)
121 else:
122 s = '{}[id={}, inputId={}]\n'.format(self.name, self.id, self.inputId)
123
124 for key, value in self.parameters.items():
125 if self.ELEMENTNAME == 'Operation':
126 s += ' {}: {}\n'.format(key, value)
127 else:
128 s += ' {}: {}\n'.format(key, value)
129
130 for conf in self.operations:
131 s += str(conf)
132
133 return s
134
135 class OperationConf(ConfBase):
136
137 ELEMENTNAME = 'Operation'
138 xml_labels = ['id', 'name']
139
140 def setup(self, id, name, priority, project_id, err_queue):
141
142 self.id = str(id)
143 self.project_id = project_id
144 self.name = name
145 self.type = 'other'
146 self.err_queue = err_queue
147
148 def readXml(self, element, project_id, err_queue):
149
150 self.id = element.get('id')
151 self.name = element.get('name')
152 self.type = 'other'
153 self.project_id = str(project_id)
154 self.err_queue = err_queue
155
156 for elm in element.iter('Parameter'):
157 self.addParameter(elm.get('name'), elm.get('value'))
158
159 def createObject(self):
160
161 className = eval(self.name)
162
163 if 'Plot' in self.name or 'Writer' in self.name or 'Send' in self.name or 'print' in self.name:
164 kwargs = self.getKwargs()
165 opObj = className(self.id, self.id, self.project_id, self.err_queue, **kwargs)
166 opObj.start()
167 self.type = 'external'
168 else:
169 opObj = className()
170
171 self.object = opObj
172 return opObj
173
174 class ProcUnitConf(ConfBase):
175
176 ELEMENTNAME = 'ProcUnit'
177 xml_labels = ['id', 'inputId', 'name']
178
179 def setup(self, project_id, id, name, datatype, inputId, err_queue):
180 '''
181 '''
182
183 if datatype == None and name == None:
184 raise ValueError('datatype or name should be defined')
185
186 if name == None:
187 if 'Proc' in datatype:
188 name = datatype
189 else:
190 name = '%sProc' % (datatype)
191
192 if datatype == None:
193 datatype = name.replace('Proc', '')
194
195 self.id = str(id)
196 self.project_id = project_id
197 self.name = name
198 self.datatype = datatype
199 self.inputId = inputId
200 self.err_queue = err_queue
201 self.operations = []
202 self.parameters = {}
203
204 def removeOperation(self, id):
205
206 i = [1 if x.id==id else 0 for x in self.operations]
207 self.operations.pop(i.index(1))
208
209 def getOperation(self, id):
210
211 for conf in self.operations:
212 if conf.id == id:
213 return conf
214
215 def addOperation(self, name, optype='self'):
216 '''
217 '''
218
219 id = self.getNewId()
220 conf = OperationConf()
221 conf.setup(id, name=name, priority='0', project_id=self.project_id, err_queue=self.err_queue)
222 self.operations.append(conf)
223
224 return conf
225
226 def readXml(self, element, project_id, err_queue):
227
228 self.id = element.get('id')
229 self.name = element.get('name')
230 self.inputId = None if element.get('inputId') == 'None' else element.get('inputId')
231 self.datatype = element.get('datatype', self.name.replace(self.ELEMENTNAME.replace('Unit', ''), ''))
232 self.project_id = str(project_id)
233 self.err_queue = err_queue
234 self.operations = []
235 self.parameters = {}
236
237 for elm in element:
238 if elm.tag == 'Parameter':
239 self.addParameter(elm.get('name'), elm.get('value'))
240 elif elm.tag == 'Operation':
241 conf = OperationConf()
242 conf.readXml(elm, project_id, err_queue)
243 self.operations.append(conf)
244
245 def createObjects(self):
246 '''
247 Instancia de unidades de procesamiento.
248 '''
249
250 className = eval(self.name)
251 kwargs = self.getKwargs()
252 procUnitObj = className()
253 procUnitObj.name = self.name
254 log.success('creating process...', self.name)
255
256 for conf in self.operations:
257
258 opObj = conf.createObject()
259
260 log.success('adding operation: {}, type:{}'.format(
261 conf.name,
262 conf.type), self.name)
263
264 procUnitObj.addOperation(conf, opObj)
265
266 self.object = procUnitObj
267
268 def run(self):
269 '''
270 '''
271
272 return self.object.call(**self.getKwargs())
273
274
275 class ReadUnitConf(ProcUnitConf):
276
277 ELEMENTNAME = 'ReadUnit'
278
279 def __init__(self):
280
281 self.id = None
282 self.datatype = None
283 self.name = None
284 self.inputId = None
285 self.operations = []
286 self.parameters = {}
287
288 def setup(self, project_id, id, name, datatype, err_queue, path='', startDate='', endDate='',
289 startTime='', endTime='', server=None, **kwargs):
290
291 if datatype == None and name == None:
292 raise ValueError('datatype or name should be defined')
293 if name == None:
294 if 'Reader' in datatype:
295 name = datatype
296 datatype = name.replace('Reader','')
297 else:
298 name = '{}Reader'.format(datatype)
299 if datatype == None:
300 if 'Reader' in name:
301 datatype = name.replace('Reader','')
302 else:
303 datatype = name
304 name = '{}Reader'.format(name)
305
306 self.id = id
307 self.project_id = project_id
308 self.name = name
309 self.datatype = datatype
310 self.err_queue = err_queue
311
312 self.addParameter(name='path', value=path)
313 self.addParameter(name='startDate', value=startDate)
314 self.addParameter(name='endDate', value=endDate)
315 self.addParameter(name='startTime', value=startTime)
316 self.addParameter(name='endTime', value=endTime)
317
318 for key, value in kwargs.items():
319 self.addParameter(name=key, value=value)
320
321
322 class Project(Process):
323 """API to create signal chain projects"""
324
325 ELEMENTNAME = 'Project'
326
327 def __init__(self, name=''):
328
329 Process.__init__(self)
330 self.id = '1'
331 if name:
332 self.name = '{} ({})'.format(Process.__name__, name)
333 self.filename = None
334 self.description = None
335 self.email = None
336 self.alarm = []
337 self.configurations = {}
338 # self.err_queue = Queue()
339 self.err_queue = None
340 self.started = False
341
342 def getNewId(self):
343
344 idList = list(self.configurations.keys())
345 id = int(self.id) * 10
346
347 while True:
348 id += 1
349
350 if str(id) in idList:
351 continue
352
353 break
354
355 return str(id)
356
357 def updateId(self, new_id):
358
359 self.id = str(new_id)
360
361 keyList = list(self.configurations.keys())
362 keyList.sort()
363
364 n = 1
365 new_confs = {}
366
367 for procKey in keyList:
368
369 conf = self.configurations[procKey]
370 idProcUnit = str(int(self.id) * 10 + n)
371 conf.updateId(idProcUnit)
372 new_confs[idProcUnit] = conf
373 n += 1
374
375 self.configurations = new_confs
376
377 def setup(self, id=1, name='', description='', email=None, alarm=[]):
378
379 self.id = str(id)
380 self.description = description
381 self.email = email
382 self.alarm = alarm
383 if name:
384 self.name = '{} ({})'.format(Process.__name__, name)
385
386 def update(self, **kwargs):
387
388 for key, value in kwargs.items():
389 setattr(self, key, value)
390
391 def clone(self):
392
393 p = Project()
394 p.id = self.id
395 p.name = self.name
396 p.description = self.description
397 p.configurations = self.configurations.copy()
398
399 return p
400
401 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
402
403 '''
404 '''
405
406 if id is None:
407 idReadUnit = self.getNewId()
408 else:
409 idReadUnit = str(id)
410
411 conf = ReadUnitConf()
412 conf.setup(self.id, idReadUnit, name, datatype, self.err_queue, **kwargs)
413 self.configurations[conf.id] = conf
414
415 return conf
416
417 def addProcUnit(self, id=None, inputId='0', datatype=None, name=None):
418
419 '''
420 '''
421
422 if id is None:
423 idProcUnit = self.getNewId()
424 else:
425 idProcUnit = id
426
427 conf = ProcUnitConf()
428 conf.setup(self.id, idProcUnit, name, datatype, inputId, self.err_queue)
429 self.configurations[conf.id] = conf
430
431 return conf
432
433 def removeProcUnit(self, id):
434
435 if id in self.configurations:
436 self.configurations.pop(id)
437
438 def getReadUnit(self):
439
440 for obj in list(self.configurations.values()):
441 if obj.ELEMENTNAME == 'ReadUnit':
442 return obj
443
444 return None
445
446 def getProcUnit(self, id):
447
448 return self.configurations[id]
449
450 def getUnits(self):
451
452 keys = list(self.configurations)
453 keys.sort()
454
455 for key in keys:
456 yield self.configurations[key]
457
458 def updateUnit(self, id, **kwargs):
459
460 conf = self.configurations[id].update(**kwargs)
461
462 def makeXml(self):
463
464 xml = Element('Project')
465 xml.set('id', str(self.id))
466 xml.set('name', self.name)
467 xml.set('description', self.description)
468
469 for conf in self.configurations.values():
470 conf.makeXml(xml)
471
472 self.xml = xml
473
474 def writeXml(self, filename=None):
475
476 if filename == None:
477 if self.filename:
478 filename = self.filename
479 else:
480 filename = 'schain.xml'
481
482 if not filename:
483 print('filename has not been defined. Use setFilename(filename) for do it.')
484 return 0
485
486 abs_file = os.path.abspath(filename)
487
488 if not os.access(os.path.dirname(abs_file), os.W_OK):
489 print('No write permission on %s' % os.path.dirname(abs_file))
490 return 0
491
492 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
493 print('File %s already exists and it could not be overwriten' % abs_file)
494 return 0
495
496 self.makeXml()
497
498 ElementTree(self.xml).write(abs_file, method='xml')
499
500 self.filename = abs_file
501
502 return 1
503
504 def readXml(self, filename):
505
506 abs_file = os.path.abspath(filename)
507
508 self.configurations = {}
509
510 try:
511 self.xml = ElementTree().parse(abs_file)
512 except:
513 log.error('Error reading %s, verify file format' % filename)
514 return 0
515
516 self.id = self.xml.get('id')
517 self.name = self.xml.get('name')
518 self.description = self.xml.get('description')
519
520 for element in self.xml:
521 if element.tag == 'ReadUnit':
522 conf = ReadUnitConf()
523 conf.readXml(element, self.id, self.err_queue)
524 self.configurations[conf.id] = conf
525 elif element.tag == 'ProcUnit':
526 conf = ProcUnitConf()
527 input_proc = self.configurations[element.get('inputId')]
528 conf.readXml(element, self.id, self.err_queue)
529 self.configurations[conf.id] = conf
530
531 self.filename = abs_file
532
533 return 1
534
535 def __str__(self):
536
537 text = '\nProject[id=%s, name=%s, description=%s]\n\n' % (
538 self.id,
539 self.name,
540 self.description,
541 )
542
543 for conf in self.configurations.values():
544 text += '{}'.format(conf)
545
546 return text
547
548 def createObjects(self):
549
550 keys = list(self.configurations.keys())
551 keys.sort()
552 for key in keys:
553 conf = self.configurations[key]
554 conf.createObjects()
555 if conf.inputId is not None:
556 conf.object.setInput(self.configurations[conf.inputId].object)
557
558 def monitor(self):
559
560 t = Thread(target=self._monitor, args=(self.err_queue, self.ctx))
561 t.start()
562
563 def _monitor(self, queue, ctx):
564
565 import socket
566
567 procs = 0
568 err_msg = ''
569
570 while True:
571 msg = queue.get()
572 if '#_start_#' in msg:
573 procs += 1
574 elif '#_end_#' in msg:
575 procs -=1
576 else:
577 err_msg = msg
578
579 if procs == 0 or 'Traceback' in err_msg:
580 break
581 time.sleep(0.1)
582
583 if '|' in err_msg:
584 name, err = err_msg.split('|')
585 if 'SchainWarning' in err:
586 log.warning(err.split('SchainWarning:')[-1].split('\n')[0].strip(), name)
587 elif 'SchainError' in err:
588 log.error(err.split('SchainError:')[-1].split('\n')[0].strip(), name)
589 else:
590 log.error(err, name)
591 else:
592 name, err = self.name, err_msg
593
594 time.sleep(1)
595
596 ctx.term()
597
598 message = ''.join(err)
599
600 if err_msg:
601 subject = 'SChain v%s: Error running %s\n' % (
602 schainpy.__version__, self.name)
603
604 subtitle = 'Hostname: %s\n' % socket.gethostbyname(
605 socket.gethostname())
606 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
607 subtitle += 'Configuration file: %s\n' % self.filename
608 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
609
610 readUnitConfObj = self.getReadUnit()
611 if readUnitConfObj:
612 subtitle += '\nInput parameters:\n'
613 subtitle += '[Data path = %s]\n' % readUnitConfObj.parameters['path']
614 subtitle += '[Start date = %s]\n' % readUnitConfObj.parameters['startDate']
615 subtitle += '[End date = %s]\n' % readUnitConfObj.parameters['endDate']
616 subtitle += '[Start time = %s]\n' % readUnitConfObj.parameters['startTime']
617 subtitle += '[End time = %s]\n' % readUnitConfObj.parameters['endTime']
618
619 a = Alarm(
620 modes=self.alarm,
621 email=self.email,
622 message=message,
623 subject=subject,
624 subtitle=subtitle,
625 filename=self.filename
626 )
627
628 a.start()
629
630 def setFilename(self, filename):
631
632 self.filename = filename
633
634 def runProcs(self):
635
636 err = False
637 n = len(self.configurations)
638
639 while not err:
640 for conf in self.getUnits():
641 ok = conf.run()
642 if ok == 'Error':
643 n -= 1
644 continue
645 elif not ok:
646 break
647 if n == 0:
648 err = True
649
650 def run(self):
651
652 log.success('\nStarting Project {} [id={}]'.format(self.name, self.id), tag='')
653 self.started = True
654 self.start_time = time.time()
655 self.createObjects()
656 self.runProcs()
657 log.success('{} Done (Time: {:4.2f}s)'.format(
658 self.name,
659 time.time()-self.start_time), '')
Show file before | Show file after | Hide comments
@@ -24,7 +24,7 from schainpy.model.data.jrodata import Voltage
24 24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
25 25 from numpy import imag
26 26
27 @MPDecorator
27
28 28 class AMISRReader(ProcessingUnit):
29 29 '''
30 30 classdocs
@@ -82,6 +82,7 class AMISRReader(ProcessingUnit):
82 82 self.dataOut = Voltage()
83 83 self.dataOut.error=False
84 84
85
85 86 def setup(self,path=None,
86 87 startDate=None,
87 88 endDate=None,
@@ -95,7 +96,7 class AMISRReader(ProcessingUnit):
95 96 nBaud = 0,
96 97 online=False):
97 98
98 #print ("T",path)
99
99 100
100 101 self.timezone = timezone
101 102 self.all = all
@@ -126,20 +127,24 class AMISRReader(ProcessingUnit):
126 127 Add code
127 128 '''
128 129 self.isConfig = True
129
130 # print("Setup Done")
130 131 pass
131 132
132 133
133 134 def readAMISRHeader(self,fp):
134 135 header = 'Raw11/Data/RadacHeader'
135 136 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
136 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
137 self.trueBeams = self.beamcodeFile.split("\n")
138 self.trueBeams.pop()#remove last
139 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
140 self.beamCode = [int(x, 16) for x in self.realBeamCode]
137 if (self.startDate> datetime.date(2021, 7, 15)): #Se cambió la forma de extracción de Apuntes el 17
138 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
139 self.trueBeams = self.beamcodeFile.split("\n")
140 self.trueBeams.pop()#remove last
141 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
142 self.beamCode = [int(x, 16) for x in self.realBeamCode]
143 else:
144 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
145 self.beamCode = _beamCode[0,:]
146
141 147
142 #self.beamCode = fp.get('Raw11/Data/Beamcodes') # NUMBER OF CHANNELS AND IDENTIFY POSITION TO CREATE A FILE WITH THAT INFO
143 148 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
144 149 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
145 150 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
@@ -625,9 +630,6 class AMISRReader(ProcessingUnit):
625 630 diffUTC = 1.8e4 #UTC diference from peru in seconds --Joab
626 631 diffUTC = 0
627 632 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
628 #cambio posible 18/02/2020
629
630
631 633
632 634 #print("utc :",indexblock," __ ",t_comp)
633 635 #print(numpy.shape(self.timeset))
@@ -648,7 +650,7 class AMISRReader(ProcessingUnit):
648 650 '''
649 651 This method will be called many times so here you should put all your code
650 652 '''
651
653 #print("running kamisr")
652 654 if not self.isConfig:
653 655 self.setup(**kwargs)
654 656 self.isConfig = True
Show file before | Show file after | Hide comments
@@ -75,7 +75,7 class SpectraReader(JRODataReader, ProcessingUnit):
75 75
76 76 self.pts2read_SelfSpectra = 0
77 77 self.pts2read_CrossSpectra = 0
78 self.pts2read_DCchannels = 0
78 self.pts2read_DCchannels = 0
79 79 self.ext = ".pdata"
80 80 self.optchar = "P"
81 81 self.basicHeaderObj = BasicHeader(LOCALTIME)
@@ -162,7 +162,7 class SpectraReader(JRODataReader, ProcessingUnit):
162 162 Exceptions:
163 163 Si un bloque leido no es un bloque valido
164 164 """
165
165
166 166 fpointer = self.fp.tell()
167 167
168 168 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
@@ -364,7 +364,7 class SpectraWriter(JRODataWriter, Operation):
364 364 data.tofile(self.fp)
365 365
366 366 if self.data_cspc is not None:
367
367
368 368 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
369 369 data = numpy.zeros( numpy.shape(cspc), self.dtype )
370 370 #print 'data.shape', self.shape_cspc_Buffer
@@ -376,7 +376,7 class SpectraWriter(JRODataWriter, Operation):
376 376 data.tofile(self.fp)
377 377
378 378 if self.data_dc is not None:
379
379
380 380 dc = self.data_dc
381 381 data = numpy.zeros( numpy.shape(dc), self.dtype )
382 382 data['real'] = dc.real
@@ -524,4 +524,4 class SpectraWriter(JRODataWriter, Operation):
524 524
525 525 self.processingHeaderObj.processFlags = self.getProcessFlags()
526 526
527 self.setBasicHeader() No newline at end of file
527 self.setBasicHeader()
Show file before | Show file after | Hide comments
@@ -174,7 +174,7 def MPDecorator(BaseClass):
174 174 self.name = '{}{}'.format(self.CODE.upper(), 'Plot')
175 175
176 176 self.start_time = time.time()
177 self.err_queue = args[3]
177 #self.err_queue = args[2]
178 178 self.queue = Queue(maxsize=1)
179 179 self.myrun = BaseClass.run
180 180
Show file before | Show file after | Hide comments
@@ -1,4 +1,4
1 #!/usr/bin/env python
1
2 2 import os, sys
3 3 import time
4 4 import datetime
@@ -20,10 +20,10 def main():
20 20 ymax = '300'
21 21 dbmin = '45' #'60'#'55' #'40' #noise esf eej
22 22 dbmax = '65' #'70' #'55'
23 showSPC = '0' #view plot Spectra
24 showRTI = '0' #view plot RTI
25 showNOISE = '0' #view plot NOISE
26 localtime='0' #para ajustar el horario en las gráficas '0' para dejar en utc
23 showSPC = '1' #view plot Spectra
24 showRTI = '1' #view plot RTI
25 showNOISE = '1' #view plot NOISE
26 localtime='1' #para ajustar el horario en las gráficas '0' para dejar en utc
27 27 code = '1,-1,-1,-1,1,1,1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,1,-1,1,1,-1,1'
28 28 nCode = '1'
29 29 nBaud = '28'
@@ -31,10 +31,11 def main():
31 31 today = time.strftime("%Y/%m/%d")
32 32 #startDate=today
33 33 #endDate=today
34 startDate='2019/12/16'
35 endDate='2019/12/16'
34 startDate='2021/07/11'
35 endDate='2021/07/11'
36 36 #inPath= '/home/soporte/dataAMISR_test/'
37 37 inPath= '/home/soporte/dataAMISR/'
38 inPath= '/media/soporte/UARS_4T_D02/AMISR_DATA/2021/'
38 39 #inPath = '/mnt/data_amisr'
39 40 outPath = '/home/soporte/Data/EEJ'
40 41
@@ -64,19 +65,22 def main():
64 65 endDate=endDate, #endDate '2014/10/07',
65 66 startTime='07:01:30',#'07:00:00',
66 67 endTime='19:00:00',#'15:00:00',
67 walk=0,
68 walk=1,
68 69 code = code,
69 70 nCode = nCode,
70 71 nBaud = nBaud,
71 timezone='ut',
72 timezone='lt',
72 73 online=0)
73 74
75
74 76 #AMISR Processing Unit
75 77 ##.......................................................................................
76 78 ##.......................................................................................
77 79 procUnitConfObj0 = controllerObj.addProcUnit(datatype='VoltageProc', inputId=readUnitConfObj.getId())
78 opObj10 = procUnitConfObj0.addOperation(name='setRadarFrequency')
79 opObj10.addParameter(name='frequency', value='445e6', format='float')
80 opObj10 = procUnitConfObj0.addOperation(name='setAttribute')
81 opObj10.addParameter(name='frequency', value='445.09e6')
82 # opObj10 = procUnitConfObj0.addOperation(name='setRadarFrequency')
83 # opObj10.addParameter(name='frequency', value='445e6', format='float')
80 84
81 85
82 86 opObj01 = procUnitConfObj0.addOperation(name='Decoder', optype='other')
@@ -86,8 +90,8 def main():
86 90 opObj01.addParameter(name='osamp', value=nosamp, format='int')
87 91
88 92
89 opObj02 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
90 opObj02.addParameter(name='n', value='2', format='int')
93 # opObj02 = procUnitConfObj0.addOperation(name='CohInt', optype='other')
94 # opObj02.addParameter(name='n', value='2', format='int')
91 95
92 96
93 97
@@ -106,9 +110,9 def main():
106 110 ##.......................................................................................
107 111 ##.......................................................................................
108 112
109 opObj13 = procUnitConfObj1.addOperation(name='getNoise' , optype ='self')
110 opObj13.addParameter(name='minHei', value='100', format='float')
111 opObj13.addParameter(name='maxHei', value='280', format='float')
113 # opObj13 = procUnitConfObj1.addOperation(name='getNoise' , optype ='self')
114 # opObj13.addParameter(name='minHei', value='100', format='float')
115 # opObj13.addParameter(name='maxHei', value='280', format='float')
112 116
113 117
114 118 #
@@ -138,21 +142,21 def main():
138 142 opObj14.addParameter(name='id', value='3', format='int')
139 143 opObj14.addParameter(name='wintitle', value='title0', format='str')
140 144 opObj14.addParameter(name='showprofile', value='0', format='int')
141 opObj14.addParameter(name='xmin', value=xmin, format='int')
142 opObj14.addParameter(name='xmax', value=xmax, format='int')
145 opObj14.addParameter(name='tmin', value=xmin, format='int')
146 opObj14.addParameter(name='tmax', value=xmax, format='int')
143 147 opObj14.addParameter(name='ymin', value=dbmin, format='int')
144 148 opObj14.addParameter(name='ymax', value=dbmax, format='int')
145 149 opObj14.addParameter(name='save', value=outPath, format='str')
146 150 opObj14.addParameter(name='localtime', value=localtime,format='int')
147 151 opObj14.addParameter(name='show', value = showNOISE, format='int')
148 152
149
153 #
150 154 opObj15 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
151 155 opObj15.addParameter(name='id', value='2', format='int')
152 156 opObj15.addParameter(name='localtime', value=localtime,format='int')
153 157 opObj15.addParameter(name='wintitle', value='RTI', format='str')
154 opObj15.addParameter(name='xmin', value=xmin, format='int')
155 opObj15.addParameter(name='xmax', value=xmax, format='int') #max value =23
158 opObj15.addParameter(name='tmin', value=xmin, format='int')
159 opObj15.addParameter(name='tmax', value=xmax, format='int') #max value =23
156 160 opObj15.addParameter(name='ymin', value=ymin, format='int')
157 161 opObj15.addParameter(name='zmin', value=dbmin, format='int')
158 162 opObj15.addParameter(name='zmax', value=dbmax, format='int')
@@ -170,16 +174,15 def main():
170 174 opObj16 = procUnitConfObj2.addOperation(name='SpectralMoments', optype='other')
171 175
172 176
173 #Using ParamWriter::::
177 #Using HDFWriter::::
174 178 ##.......................................................................................
175 179 ##.......................................................................................
176 opObj17 = procUnitConfObj2.addOperation(name='ParamWriter', optype='external')
180 opObj17 = procUnitConfObj2.addOperation(name='HDFWriter', optype='external')
177 181 opObj17.addParameter(name='path', value=outPath)
178 182 opObj17.addParameter(name='blocksPerFile', value='10', format='int')
179 183 opObj17.addParameter(name='metadataList',value='type,inputUnit,heightList',format='list')
180 184 opObj17.addParameter(name='dataList',value='moments,data_SNR,utctime',format='list')
181 opObj17.addParameter(name='mode',value='1',format='int') #'0' channels, '1' parameters, '3' table (for meteors)
182 ##opObj17.addParameter(name='setType', value ='anything', format='str')#no usar
185
183 186
184 187
185 188 ##.......................................................................................
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now