##// END OF EJS Templates
schain-jc
RSS
Multiprocessing for heispectra (Fits) all working
George Yong -
r1191:f1f11d1fc586
parent child
Show More
Side by Side Unified
Context file:
r1191:f1f11d1fc586 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1263 +1,1254
1 1 '''
2 2 Updated on January , 2018, for multiprocessing purposes
3 3 Author: Sergio Cortez
4 4 Created on September , 2012
5 5 '''
6 6 from platform import python_version
7 7 import sys
8 8 import ast
9 9 import datetime
10 10 import traceback
11 11 import math
12 12 import time
13 13 import zmq
14 14 from multiprocessing import Process, cpu_count
15 15 from threading import Thread
16 16 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
17 17 from xml.dom import minidom
18 18
19 19
20 20 from schainpy.admin import Alarm, SchainWarning
21
22 ### Temporary imports!!!
23 # from schainpy.model import *
24 from schainpy.model.io import *
25 from schainpy.model.graphics import *
26 from schainpy.model.proc.jroproc_base import *
27 from schainpy.model.proc.bltrproc_parameters import *
28 from schainpy.model.proc.jroproc_spectra import *
29 from schainpy.model.proc.jroproc_voltage import *
30 from schainpy.model.proc.jroproc_parameters import *
31 from schainpy.model.utils.jroutils_publish import *
21 from schainpy.model import *
32 22 from schainpy.utils import log
33 ###
23
34 24
35 25 DTYPES = {
36 26 'Voltage': '.r',
37 27 'Spectra': '.pdata'
38 28 }
39 29
40 30
41 31 def MPProject(project, n=cpu_count()):
42 32 '''
43 33 Project wrapper to run schain in n processes
44 34 '''
45 35
46 36 rconf = project.getReadUnitObj()
47 37 op = rconf.getOperationObj('run')
48 38 dt1 = op.getParameterValue('startDate')
49 39 dt2 = op.getParameterValue('endDate')
50 40 tm1 = op.getParameterValue('startTime')
51 41 tm2 = op.getParameterValue('endTime')
52 42 days = (dt2 - dt1).days
53 43
54 44 for day in range(days + 1):
55 45 skip = 0
56 46 cursor = 0
57 47 processes = []
58 48 dt = dt1 + datetime.timedelta(day)
59 49 dt_str = dt.strftime('%Y/%m/%d')
60 50 reader = JRODataReader()
61 51 paths, files = reader.searchFilesOffLine(path=rconf.path,
62 52 startDate=dt,
63 53 endDate=dt,
64 54 startTime=tm1,
65 55 endTime=tm2,
66 56 ext=DTYPES[rconf.datatype])
67 57 nFiles = len(files)
68 58 if nFiles == 0:
69 59 continue
70 60 skip = int(math.ceil(nFiles / n))
71 61 while nFiles > cursor * skip:
72 62 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
73 63 skip=skip)
74 64 p = project.clone()
75 65 p.start()
76 66 processes.append(p)
77 67 cursor += 1
78 68
79 69 def beforeExit(exctype, value, trace):
80 70 for process in processes:
81 71 process.terminate()
82 72 process.join()
83 73 print(traceback.print_tb(trace))
84 74
85 75 sys.excepthook = beforeExit
86 76
87 77 for process in processes:
88 78 process.join()
89 79 process.terminate()
90 80
91 81 time.sleep(3)
92 82
93 83 def wait(context):
94 84
95 85 time.sleep(1)
96 86 c = zmq.Context()
97 87 receiver = c.socket(zmq.SUB)
98 88 receiver.connect('ipc:///tmp/schain_{}_pub'.format(self.id))
99 89 receiver.setsockopt(zmq.SUBSCRIBE, self.id.encode())
100 90 msg = receiver.recv_multipart()[1]
101 91 context.terminate()
102 92
103 93 class ParameterConf():
104 94
105 95 id = None
106 96 name = None
107 97 value = None
108 98 format = None
109 99
110 100 __formated_value = None
111 101
112 102 ELEMENTNAME = 'Parameter'
113 103
114 104 def __init__(self):
115 105
116 106 self.format = 'str'
117 107
118 108 def getElementName(self):
119 109
120 110 return self.ELEMENTNAME
121 111
122 112 def getValue(self):
123 113
124 114 value = self.value
125 115 format = self.format
126 116
127 117 if self.__formated_value != None:
128 118
129 119 return self.__formated_value
130 120
131 121 if format == 'obj':
132 122 return value
133 123
134 124 if format == 'str':
135 125 self.__formated_value = str(value)
136 126 return self.__formated_value
137 127
138 128 if value == '':
139 129 raise ValueError('%s: This parameter value is empty' % self.name)
140 130
141 131 if format == 'list':
142 132 strList = value.split(',')
143 133
144 134 self.__formated_value = strList
145 135
146 136 return self.__formated_value
147 137
148 138 if format == 'intlist':
149 139 '''
150 140 Example:
151 141 value = (0,1,2)
152 142 '''
153 143
154 144 new_value = ast.literal_eval(value)
155 145
156 146 if type(new_value) not in (tuple, list):
157 147 new_value = [int(new_value)]
158 148
159 149 self.__formated_value = new_value
160 150
161 151 return self.__formated_value
162 152
163 153 if format == 'floatlist':
164 154 '''
165 155 Example:
166 156 value = (0.5, 1.4, 2.7)
167 157 '''
168 158
169 159 new_value = ast.literal_eval(value)
170 160
171 161 if type(new_value) not in (tuple, list):
172 162 new_value = [float(new_value)]
173 163
174 164 self.__formated_value = new_value
175 165
176 166 return self.__formated_value
177 167
178 168 if format == 'date':
179 169 strList = value.split('/')
180 170 intList = [int(x) for x in strList]
181 171 date = datetime.date(intList[0], intList[1], intList[2])
182 172
183 173 self.__formated_value = date
184 174
185 175 return self.__formated_value
186 176
187 177 if format == 'time':
188 178 strList = value.split(':')
189 179 intList = [int(x) for x in strList]
190 180 time = datetime.time(intList[0], intList[1], intList[2])
191 181
192 182 self.__formated_value = time
193 183
194 184 return self.__formated_value
195 185
196 186 if format == 'pairslist':
197 187 '''
198 188 Example:
199 189 value = (0,1),(1,2)
200 190 '''
201 191
202 192 new_value = ast.literal_eval(value)
203 193
204 194 if type(new_value) not in (tuple, list):
205 195 raise ValueError('%s has to be a tuple or list of pairs' % value)
206 196
207 197 if type(new_value[0]) not in (tuple, list):
208 198 if len(new_value) != 2:
209 199 raise ValueError('%s has to be a tuple or list of pairs' % value)
210 200 new_value = [new_value]
211 201
212 202 for thisPair in new_value:
213 203 if len(thisPair) != 2:
214 204 raise ValueError('%s has to be a tuple or list of pairs' % value)
215 205
216 206 self.__formated_value = new_value
217 207
218 208 return self.__formated_value
219 209
220 210 if format == 'multilist':
221 211 '''
222 212 Example:
223 213 value = (0,1,2),(3,4,5)
224 214 '''
225 215 multiList = ast.literal_eval(value)
226 216
227 217 if type(multiList[0]) == int:
228 218 multiList = ast.literal_eval('(' + value + ')')
229 219
230 220 self.__formated_value = multiList
231 221
232 222 return self.__formated_value
233 223
234 224 if format == 'bool':
235 225 value = int(value)
236 226
237 227 if format == 'int':
238 228 value = float(value)
239 229
240 230 format_func = eval(format)
241 231
242 232 self.__formated_value = format_func(value)
243 233
244 234 return self.__formated_value
245 235
246 236 def updateId(self, new_id):
247 237
248 238 self.id = str(new_id)
249 239
250 240 def setup(self, id, name, value, format='str'):
251 241 self.id = str(id)
252 242 self.name = name
253 243 if format == 'obj':
254 244 self.value = value
255 245 else:
256 246 self.value = str(value)
257 247 self.format = str.lower(format)
258 248
259 249 self.getValue()
260 250
261 251 return 1
262 252
263 253 def update(self, name, value, format='str'):
264 254
265 255 self.name = name
266 256 self.value = str(value)
267 257 self.format = format
268 258
269 259 def makeXml(self, opElement):
270 260 if self.name not in ('queue',):
271 261 parmElement = SubElement(opElement, self.ELEMENTNAME)
272 262 parmElement.set('id', str(self.id))
273 263 parmElement.set('name', self.name)
274 264 parmElement.set('value', self.value)
275 265 parmElement.set('format', self.format)
276 266
277 267 def readXml(self, parmElement):
278 268
279 269 self.id = parmElement.get('id')
280 270 self.name = parmElement.get('name')
281 271 self.value = parmElement.get('value')
282 272 self.format = str.lower(parmElement.get('format'))
283 273
284 274 # Compatible with old signal chain version
285 275 if self.format == 'int' and self.name == 'idfigure':
286 276 self.name = 'id'
287 277
288 278 def printattr(self):
289 279
290 280 print('Parameter[%s]: name = %s, value = %s, format = %s, project_id = %s' % (self.id, self.name, self.value, self.format, self.project_id))
291 281
292 282 class OperationConf():
293 283
294 284 ELEMENTNAME = 'Operation'
295 285
296 286 def __init__(self):
297 287
298 288 self.id = '0'
299 289 self.name = None
300 290 self.priority = None
301 291 self.topic = None
302 292
303 293 def __getNewId(self):
304 294
305 295 return int(self.id) * 10 + len(self.parmConfObjList) + 1
306 296
307 297 def getId(self):
308 298 return self.id
309 299
310 300 def updateId(self, new_id):
311 301
312 302 self.id = str(new_id)
313 303
314 304 n = 1
315 305 for parmObj in self.parmConfObjList:
316 306
317 307 idParm = str(int(new_id) * 10 + n)
318 308 parmObj.updateId(idParm)
319 309
320 310 n += 1
321 311
322 312 def getElementName(self):
323 313
324 314 return self.ELEMENTNAME
325 315
326 316 def getParameterObjList(self):
327 317
328 318 return self.parmConfObjList
329 319
330 320 def getParameterObj(self, parameterName):
331 321
332 322 for parmConfObj in self.parmConfObjList:
333 323
334 324 if parmConfObj.name != parameterName:
335 325 continue
336 326
337 327 return parmConfObj
338 328
339 329 return None
340 330
341 331 def getParameterObjfromValue(self, parameterValue):
342 332
343 333 for parmConfObj in self.parmConfObjList:
344 334
345 335 if parmConfObj.getValue() != parameterValue:
346 336 continue
347 337
348 338 return parmConfObj.getValue()
349 339
350 340 return None
351 341
352 342 def getParameterValue(self, parameterName):
353 343
354 344 parameterObj = self.getParameterObj(parameterName)
355 345
356 346 # if not parameterObj:
357 347 # return None
358 348
359 349 value = parameterObj.getValue()
360 350
361 351 return value
362 352
363 353 def getKwargs(self):
364 354
365 355 kwargs = {}
366 356
367 357 for parmConfObj in self.parmConfObjList:
368 358 if self.name == 'run' and parmConfObj.name == 'datatype':
369 359 continue
370 360
371 361 kwargs[parmConfObj.name] = parmConfObj.getValue()
372 362
373 363 return kwargs
374 364
375 365 def setup(self, id, name, priority, type, project_id):
376 366
377 367 self.id = str(id)
378 368 self.project_id = project_id
379 369 self.name = name
380 370 self.type = type
381 371 self.priority = priority
382 372 self.parmConfObjList = []
383 373
384 374 def removeParameters(self):
385 375
386 376 for obj in self.parmConfObjList:
387 377 del obj
388 378
389 379 self.parmConfObjList = []
390 380
391 381 def addParameter(self, name, value, format='str'):
392 382
393 383 if value is None:
394 384 return None
395 385 id = self.__getNewId()
396 386
397 387 parmConfObj = ParameterConf()
398 388 if not parmConfObj.setup(id, name, value, format):
399 389 return None
400 390
401 391 self.parmConfObjList.append(parmConfObj)
402 392
403 393 return parmConfObj
404 394
405 395 def changeParameter(self, name, value, format='str'):
406 396
407 397 parmConfObj = self.getParameterObj(name)
408 398 parmConfObj.update(name, value, format)
409 399
410 400 return parmConfObj
411 401
412 402 def makeXml(self, procUnitElement):
413 403
414 404 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
415 405 opElement.set('id', str(self.id))
416 406 opElement.set('name', self.name)
417 407 opElement.set('type', self.type)
418 408 opElement.set('priority', str(self.priority))
419 409
420 410 for parmConfObj in self.parmConfObjList:
421 411 parmConfObj.makeXml(opElement)
422 412
423 413 def readXml(self, opElement, project_id):
424 414
425 415 self.id = opElement.get('id')
426 416 self.name = opElement.get('name')
427 417 self.type = opElement.get('type')
428 418 self.priority = opElement.get('priority')
429 419 self.project_id = str(project_id) #yong
430 420
431 421 # Compatible with old signal chain version
432 422 # Use of 'run' method instead 'init'
433 423 if self.type == 'self' and self.name == 'init':
434 424 self.name = 'run'
435 425
436 426 self.parmConfObjList = []
437 427
438 428 parmElementList = opElement.iter(ParameterConf().getElementName())
439 429
440 430 for parmElement in parmElementList:
441 431 parmConfObj = ParameterConf()
442 432 parmConfObj.readXml(parmElement)
443 433
444 434 # Compatible with old signal chain version
445 435 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
446 436 if self.type != 'self' and self.name == 'Plot':
447 437 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
448 438 self.name = parmConfObj.value
449 439 continue
450 440
451 441 self.parmConfObjList.append(parmConfObj)
452 442
453 443 def printattr(self):
454 444
455 445 print('%s[%s]: name = %s, type = %s, priority = %s, project_id = %s' % (self.ELEMENTNAME,
456 446 self.id,
457 447 self.name,
458 448 self.type,
459 449 self.priority,
460 450 self.project_id))
461 451
462 452 for parmConfObj in self.parmConfObjList:
463 453 parmConfObj.printattr()
464 454
465 455 def createObject(self):
466 456
467 457 className = eval(self.name)
468 458
469 459 if self.type == 'other':
470 460 opObj = className()
471 461 elif self.type == 'external':
472 462 kwargs = self.getKwargs()
473 463 opObj = className(self.id, self.project_id, **kwargs)
474 464 opObj.start()
475 465
476 466 return opObj
477 467
478 468 class ProcUnitConf():
479 469
480 470 ELEMENTNAME = 'ProcUnit'
481 471
482 472 def __init__(self):
483 473
484 474 self.id = None
485 475 self.datatype = None
486 476 self.name = None
487 477 self.inputId = None
488 478 self.opConfObjList = []
489 479 self.procUnitObj = None
490 480 self.opObjDict = {}
491 481
492 482 def __getPriority(self):
493 483
494 484 return len(self.opConfObjList) + 1
495 485
496 486 def __getNewId(self):
497 487
498 488 return int(self.id) * 10 + len(self.opConfObjList) + 1
499 489
500 490 def getElementName(self):
501 491
502 492 return self.ELEMENTNAME
503 493
504 494 def getId(self):
505 495
506 496 return self.id
507 497
508 498 def updateId(self, new_id):
509 499 '''
510 500 new_id = int(parentId) * 10 + (int(self.id) % 10)
511 501 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
512 502
513 503 # If this proc unit has not inputs
514 504 #if self.inputId == '0':
515 505 #new_inputId = 0
516 506
517 507 n = 1
518 508 for opConfObj in self.opConfObjList:
519 509
520 510 idOp = str(int(new_id) * 10 + n)
521 511 opConfObj.updateId(idOp)
522 512
523 513 n += 1
524 514
525 515 self.parentId = str(parentId)
526 516 self.id = str(new_id)
527 517 #self.inputId = str(new_inputId)
528 518 '''
529 519 n = 1
530 520
531 521 def getInputId(self):
532 522
533 523 return self.inputId
534 524
535 525 def getOperationObjList(self):
536 526
537 527 return self.opConfObjList
538 528
539 529 def getOperationObj(self, name=None):
540 530
541 531 for opConfObj in self.opConfObjList:
542 532
543 533 if opConfObj.name != name:
544 534 continue
545 535
546 536 return opConfObj
547 537
548 538 return None
549 539
550 540 def getOpObjfromParamValue(self, value=None):
551 541
552 542 for opConfObj in self.opConfObjList:
553 543 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 544 continue
555 545 return opConfObj
556 546 return None
557 547
558 548 def getProcUnitObj(self):
559 549
560 550 return self.procUnitObj
561 551
562 552 def setup(self, project_id, id, name, datatype, inputId):
563 553 '''
564 554 id sera el topico a publicar
565 555 inputId sera el topico a subscribirse
566 556 '''
567 557
568 558 # Compatible with old signal chain version
569 559 if datatype == None and name == None:
570 560 raise ValueError('datatype or name should be defined')
571 561
572 562 #Definir una condicion para inputId cuando sea 0
573 563
574 564 if name == None:
575 565 if 'Proc' in datatype:
576 566 name = datatype
577 567 else:
578 568 name = '%sProc' % (datatype)
579 569
580 570 if datatype == None:
581 571 datatype = name.replace('Proc', '')
582 572
583 573 self.id = str(id)
584 574 self.project_id = project_id
585 575 self.name = name
586 576 self.datatype = datatype
587 577 self.inputId = inputId
588 578 self.opConfObjList = []
589 579
590 580 self.addOperation(name='run', optype='self')
591 581
592 582 def removeOperations(self):
593 583
594 584 for obj in self.opConfObjList:
595 585 del obj
596 586
597 587 self.opConfObjList = []
598 588 self.addOperation(name='run')
599 589
600 590 def addParameter(self, **kwargs):
601 591 '''
602 592 Add parameters to 'run' operation
603 593 '''
604 594 opObj = self.opConfObjList[0]
605 595
606 596 opObj.addParameter(**kwargs)
607 597
608 598 return opObj
609 599
610 600 def addOperation(self, name, optype='self'):
611 601 '''
612 602 Actualizacion - > proceso comunicacion
613 603 En el caso de optype='self', elminar. DEfinir comuncacion IPC -> Topic
614 604 definir el tipoc de socket o comunicacion ipc++
615 605
616 606 '''
617 607
618 608 id = self.__getNewId()
619 609 priority = self.__getPriority() # Sin mucho sentido, pero puede usarse
620 610 opConfObj = OperationConf()
621 611 opConfObj.setup(id, name=name, priority=priority, type=optype, project_id=self.project_id)
622 612 self.opConfObjList.append(opConfObj)
623 613
624 614 return opConfObj
625 615
626 616 def makeXml(self, projectElement):
627 617
628 618 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
629 619 procUnitElement.set('id', str(self.id))
630 620 procUnitElement.set('name', self.name)
631 621 procUnitElement.set('datatype', self.datatype)
632 622 procUnitElement.set('inputId', str(self.inputId))
633 623
634 624 for opConfObj in self.opConfObjList:
635 625 opConfObj.makeXml(procUnitElement)
636 626
637 627 def readXml(self, upElement, project_id):
638 628
639 629 self.id = upElement.get('id')
640 630 self.name = upElement.get('name')
641 631 self.datatype = upElement.get('datatype')
642 632 self.inputId = upElement.get('inputId')
643 633 self.project_id = str(project_id)
644 634
645 635 if self.ELEMENTNAME == 'ReadUnit':
646 636 self.datatype = self.datatype.replace('Reader', '')
647 637
648 638 if self.ELEMENTNAME == 'ProcUnit':
649 639 self.datatype = self.datatype.replace('Proc', '')
650 640
651 641 if self.inputId == 'None':
652 642 self.inputId = '0'
653 643
654 644 self.opConfObjList = []
655 645
656 646 opElementList = upElement.iter(OperationConf().getElementName())
657 647
658 648 for opElement in opElementList:
659 649 opConfObj = OperationConf()
660 650 opConfObj.readXml(opElement, project_id)
661 651 self.opConfObjList.append(opConfObj)
662 652
663 653 def printattr(self):
664 654
665 655 print('%s[%s]: name = %s, datatype = %s, inputId = %s, project_id = %s' % (self.ELEMENTNAME,
666 656 self.id,
667 657 self.name,
668 658 self.datatype,
669 659 self.inputId,
670 660 self.project_id))
671 661
672 662 for opConfObj in self.opConfObjList:
673 663 opConfObj.printattr()
674 664
675 665 def getKwargs(self):
676 666
677 667 opObj = self.opConfObjList[0]
678 668 kwargs = opObj.getKwargs()
679 669
680 670 return kwargs
681 671
682 672 def createObjects(self):
683 673 '''
684 674 Instancia de unidades de procesamiento.
685 675 '''
676
686 677 className = eval(self.name)
687 678 kwargs = self.getKwargs()
688 679 procUnitObj = className(self.id, self.inputId, self.project_id, **kwargs) # necesitan saber su id y su entrada por fines de ipc
689 680 log.success('creating process...', self.name)
690 681
691 682 for opConfObj in self.opConfObjList:
692 683
693 684 if opConfObj.type == 'self' and opConfObj.name == 'run':
694 685 continue
695 686 elif opConfObj.type == 'self':
696 687 opObj = getattr(procUnitObj, opConfObj.name)
697 688 else:
698 689 opObj = opConfObj.createObject()
699 690
700 691 log.success('creating operation: {}, type:{}'.format(
701 692 opConfObj.name,
702 693 opConfObj.type), self.name)
703 694
704 695 procUnitObj.addOperation(opConfObj, opObj)
705 696
706 697 procUnitObj.start()
707 698 self.procUnitObj = procUnitObj
708 699
709 700 def close(self):
710 701
711 702 for opConfObj in self.opConfObjList:
712 703 if opConfObj.type == 'self':
713 704 continue
714 705
715 706 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
716 707 opObj.close()
717 708
718 709 self.procUnitObj.close()
719 710
720 711 return
721 712
722 713
723 714 class ReadUnitConf(ProcUnitConf):
724 715
725 716 ELEMENTNAME = 'ReadUnit'
726 717
727 718 def __init__(self):
728 719
729 720 self.id = None
730 721 self.datatype = None
731 722 self.name = None
732 723 self.inputId = None
733 724 self.opConfObjList = []
734 725
735 726 def getElementName(self):
736 727
737 728 return self.ELEMENTNAME
738 729
739 730 def setup(self, project_id, id, name, datatype, path='', startDate='', endDate='',
740 731 startTime='', endTime='', server=None, **kwargs):
741 732
742 733
743 734 '''
744 735 *****el id del proceso sera el Topico
745 736
746 737 Adicion de {topic}, si no esta presente -> error
747 738 kwargs deben ser trasmitidos en la instanciacion
748 739
749 740 '''
750 741
751 742 # Compatible with old signal chain version
752 743 if datatype == None and name == None:
753 744 raise ValueError('datatype or name should be defined')
754 745 if name == None:
755 746 if 'Reader' in datatype:
756 747 name = datatype
757 748 datatype = name.replace('Reader','')
758 749 else:
759 750 name = '{}Reader'.format(datatype)
760 751 if datatype == None:
761 752 if 'Reader' in name:
762 753 datatype = name.replace('Reader','')
763 754 else:
764 755 datatype = name
765 756 name = '{}Reader'.format(name)
766 757
767 758 self.id = id
768 759 self.project_id = project_id
769 760 self.name = name
770 761 self.datatype = datatype
771 762 if path != '':
772 763 self.path = os.path.abspath(path)
773 764 self.startDate = startDate
774 765 self.endDate = endDate
775 766 self.startTime = startTime
776 767 self.endTime = endTime
777 768 self.server = server
778 769 self.addRunOperation(**kwargs)
779 770
780 771 def update(self, **kwargs):
781 772
782 773 if 'datatype' in kwargs:
783 774 datatype = kwargs.pop('datatype')
784 775 if 'Reader' in datatype:
785 776 self.name = datatype
786 777 else:
787 778 self.name = '%sReader' % (datatype)
788 779 self.datatype = self.name.replace('Reader', '')
789 780
790 781 attrs = ('path', 'startDate', 'endDate',
791 782 'startTime', 'endTime')
792 783
793 784 for attr in attrs:
794 785 if attr in kwargs:
795 786 setattr(self, attr, kwargs.pop(attr))
796 787
797 788 self.updateRunOperation(**kwargs)
798 789
799 790 def removeOperations(self):
800 791
801 792 for obj in self.opConfObjList:
802 793 del obj
803 794
804 795 self.opConfObjList = []
805 796
806 797 def addRunOperation(self, **kwargs):
807 798
808 799 opObj = self.addOperation(name='run', optype='self')
809 800
810 801 if self.server is None:
811 802 opObj.addParameter(
812 803 name='datatype', value=self.datatype, format='str')
813 804 opObj.addParameter(name='path', value=self.path, format='str')
814 805 opObj.addParameter(
815 806 name='startDate', value=self.startDate, format='date')
816 807 opObj.addParameter(
817 808 name='endDate', value=self.endDate, format='date')
818 809 opObj.addParameter(
819 810 name='startTime', value=self.startTime, format='time')
820 811 opObj.addParameter(
821 812 name='endTime', value=self.endTime, format='time')
822 813
823 814 for key, value in list(kwargs.items()):
824 815 opObj.addParameter(name=key, value=value,
825 816 format=type(value).__name__)
826 817 else:
827 818 opObj.addParameter(name='server', value=self.server, format='str')
828 819
829 820 return opObj
830 821
831 822 def updateRunOperation(self, **kwargs):
832 823
833 824 opObj = self.getOperationObj(name='run')
834 825 opObj.removeParameters()
835 826
836 827 opObj.addParameter(name='datatype', value=self.datatype, format='str')
837 828 opObj.addParameter(name='path', value=self.path, format='str')
838 829 opObj.addParameter(
839 830 name='startDate', value=self.startDate, format='date')
840 831 opObj.addParameter(name='endDate', value=self.endDate, format='date')
841 832 opObj.addParameter(
842 833 name='startTime', value=self.startTime, format='time')
843 834 opObj.addParameter(name='endTime', value=self.endTime, format='time')
844 835
845 836 for key, value in list(kwargs.items()):
846 837 opObj.addParameter(name=key, value=value,
847 838 format=type(value).__name__)
848 839
849 840 return opObj
850 841
851 842 def readXml(self, upElement, project_id):
852 843
853 844 self.id = upElement.get('id')
854 845 self.name = upElement.get('name')
855 846 self.datatype = upElement.get('datatype')
856 847 self.project_id = str(project_id) #yong
857 848
858 849 if self.ELEMENTNAME == 'ReadUnit':
859 850 self.datatype = self.datatype.replace('Reader', '')
860 851
861 852 self.opConfObjList = []
862 853
863 854 opElementList = upElement.iter(OperationConf().getElementName())
864 855
865 856 for opElement in opElementList:
866 857 opConfObj = OperationConf()
867 858 opConfObj.readXml(opElement, project_id)
868 859 self.opConfObjList.append(opConfObj)
869 860
870 861 if opConfObj.name == 'run':
871 862 self.path = opConfObj.getParameterValue('path')
872 863 self.startDate = opConfObj.getParameterValue('startDate')
873 864 self.endDate = opConfObj.getParameterValue('endDate')
874 865 self.startTime = opConfObj.getParameterValue('startTime')
875 866 self.endTime = opConfObj.getParameterValue('endTime')
876 867
877 868
878 869 class Project(Process):
879 870
880 871 ELEMENTNAME = 'Project'
881 872
882 873 def __init__(self):
883 874
884 875 Process.__init__(self)
885 876 self.id = None
886 877 self.filename = None
887 878 self.description = None
888 879 self.email = None
889 880 self.alarm = None
890 881 self.procUnitConfObjDict = {}
891 882
892 883 def __getNewId(self):
893 884
894 885 idList = list(self.procUnitConfObjDict.keys())
895 886 id = int(self.id) * 10
896 887
897 888 while True:
898 889 id += 1
899 890
900 891 if str(id) in idList:
901 892 continue
902 893
903 894 break
904 895
905 896 return str(id)
906 897
907 898 def getElementName(self):
908 899
909 900 return self.ELEMENTNAME
910 901
911 902 def getId(self):
912 903
913 904 return self.id
914 905
915 906 def updateId(self, new_id):
916 907
917 908 self.id = str(new_id)
918 909
919 910 keyList = list(self.procUnitConfObjDict.keys())
920 911 keyList.sort()
921 912
922 913 n = 1
923 914 newProcUnitConfObjDict = {}
924 915
925 916 for procKey in keyList:
926 917
927 918 procUnitConfObj = self.procUnitConfObjDict[procKey]
928 919 idProcUnit = str(int(self.id) * 10 + n)
929 920 procUnitConfObj.updateId(idProcUnit)
930 921 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
931 922 n += 1
932 923
933 924 self.procUnitConfObjDict = newProcUnitConfObjDict
934 925
935 926 def setup(self, id=1, name='', description='', email=None, alarm=[]):
936 927
937 928 print(' ')
938 929 print('*' * 60)
939 930 print('* Starting SIGNAL CHAIN PROCESSING (Multiprocessing) v%s *' % schainpy.__version__)
940 931 print('*' * 60)
941 932 print("* Python " + python_version() + " *")
942 933 print('*' * 19)
943 934 print(' ')
944 935 self.id = str(id)
945 936 self.description = description
946 937 self.email = email
947 938 self.alarm = alarm
948 939
949 940 def update(self, **kwargs):
950 941
951 942 for key, value in list(kwargs.items()):
952 943 setattr(self, key, value)
953 944
954 945 def clone(self):
955 946
956 947 p = Project()
957 948 p.procUnitConfObjDict = self.procUnitConfObjDict
958 949 return p
959 950
960 951 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
961 952
962 953 '''
963 954 Actualizacion:
964 955 Se agrego un nuevo argumento: topic -relativo a la forma de comunicar los procesos simultaneos
965 956
966 957 * El id del proceso sera el topico al que se deben subscribir los procUnits para recibir la informacion(data)
967 958
968 959 '''
969 960
970 961 if id is None:
971 962 idReadUnit = self.__getNewId()
972 963 else:
973 964 idReadUnit = str(id)
974 965
975 966 readUnitConfObj = ReadUnitConf()
976 967 readUnitConfObj.setup(self.id, idReadUnit, name, datatype, **kwargs)
977 968 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
978 969
979 970 return readUnitConfObj
980 971
981 972 def addProcUnit(self, inputId='0', datatype=None, name=None):
982 973
983 974 '''
984 975 Actualizacion:
985 976 Se agrego dos nuevos argumentos: topic_read (lee data de otro procUnit) y topic_write(escribe o envia data a otro procUnit)
986 977 Deberia reemplazar a "inputId"
987 978
988 979 ** A fin de mantener el inputID, este sera la representaacion del topicoal que deben subscribirse. El ID propio de la intancia
989 980 (proceso) sera el topico de la publicacion, todo sera asignado de manera dinamica.
990 981
991 982 '''
992 983
993 984 idProcUnit = self.__getNewId() #Topico para subscripcion
994 985 procUnitConfObj = ProcUnitConf()
995 986 procUnitConfObj.setup(self.id, idProcUnit, name, datatype, inputId) #topic_read, topic_write,
996 987 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
997 988
998 989 return procUnitConfObj
999 990
1000 991 def removeProcUnit(self, id):
1001 992
1002 993 if id in list(self.procUnitConfObjDict.keys()):
1003 994 self.procUnitConfObjDict.pop(id)
1004 995
1005 996 def getReadUnitId(self):
1006 997
1007 998 readUnitConfObj = self.getReadUnitObj()
1008 999
1009 1000 return readUnitConfObj.id
1010 1001
1011 1002 def getReadUnitObj(self):
1012 1003
1013 1004 for obj in list(self.procUnitConfObjDict.values()):
1014 1005 if obj.getElementName() == 'ReadUnit':
1015 1006 return obj
1016 1007
1017 1008 return None
1018 1009
1019 1010 def getProcUnitObj(self, id=None, name=None):
1020 1011
1021 1012 if id != None:
1022 1013 return self.procUnitConfObjDict[id]
1023 1014
1024 1015 if name != None:
1025 1016 return self.getProcUnitObjByName(name)
1026 1017
1027 1018 return None
1028 1019
1029 1020 def getProcUnitObjByName(self, name):
1030 1021
1031 1022 for obj in list(self.procUnitConfObjDict.values()):
1032 1023 if obj.name == name:
1033 1024 return obj
1034 1025
1035 1026 return None
1036 1027
1037 1028 def procUnitItems(self):
1038 1029
1039 1030 return list(self.procUnitConfObjDict.items())
1040 1031
1041 1032 def makeXml(self):
1042 1033
1043 1034 projectElement = Element('Project')
1044 1035 projectElement.set('id', str(self.id))
1045 1036 projectElement.set('name', self.name)
1046 1037 projectElement.set('description', self.description)
1047 1038
1048 1039 for procUnitConfObj in list(self.procUnitConfObjDict.values()):
1049 1040 procUnitConfObj.makeXml(projectElement)
1050 1041
1051 1042 self.projectElement = projectElement
1052 1043
1053 1044 def writeXml(self, filename=None):
1054 1045
1055 1046 if filename == None:
1056 1047 if self.filename:
1057 1048 filename = self.filename
1058 1049 else:
1059 1050 filename = 'schain.xml'
1060 1051
1061 1052 if not filename:
1062 1053 print('filename has not been defined. Use setFilename(filename) for do it.')
1063 1054 return 0
1064 1055
1065 1056 abs_file = os.path.abspath(filename)
1066 1057
1067 1058 if not os.access(os.path.dirname(abs_file), os.W_OK):
1068 1059 print('No write permission on %s' % os.path.dirname(abs_file))
1069 1060 return 0
1070 1061
1071 1062 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1072 1063 print('File %s already exists and it could not be overwriten' % abs_file)
1073 1064 return 0
1074 1065
1075 1066 self.makeXml()
1076 1067
1077 1068 ElementTree(self.projectElement).write(abs_file, method='xml')
1078 1069
1079 1070 self.filename = abs_file
1080 1071
1081 1072 return 1
1082 1073
1083 1074 def readXml(self, filename=None):
1084 1075
1085 1076 if not filename:
1086 1077 print('filename is not defined')
1087 1078 return 0
1088 1079
1089 1080 abs_file = os.path.abspath(filename)
1090 1081
1091 1082 if not os.path.isfile(abs_file):
1092 1083 print('%s file does not exist' % abs_file)
1093 1084 return 0
1094 1085
1095 1086 self.projectElement = None
1096 1087 self.procUnitConfObjDict = {}
1097 1088
1098 1089 try:
1099 1090 self.projectElement = ElementTree().parse(abs_file)
1100 1091 except:
1101 1092 print('Error reading %s, verify file format' % filename)
1102 1093 return 0
1103 1094
1104 1095 self.project = self.projectElement.tag
1105 1096
1106 1097 self.id = self.projectElement.get('id')
1107 1098 self.name = self.projectElement.get('name')
1108 1099 self.description = self.projectElement.get('description')
1109 1100
1110 1101 readUnitElementList = self.projectElement.iter(
1111 1102 ReadUnitConf().getElementName())
1112 1103
1113 1104 for readUnitElement in readUnitElementList:
1114 1105 readUnitConfObj = ReadUnitConf()
1115 1106 readUnitConfObj.readXml(readUnitElement, self.id)
1116 1107 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1117 1108
1118 1109 procUnitElementList = self.projectElement.iter(
1119 1110 ProcUnitConf().getElementName())
1120 1111
1121 1112 for procUnitElement in procUnitElementList:
1122 1113 procUnitConfObj = ProcUnitConf()
1123 1114 procUnitConfObj.readXml(procUnitElement, self.id)
1124 1115 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1125 1116
1126 1117 self.filename = abs_file
1127 1118
1128 1119 return 1
1129 1120
1130 1121 def __str__(self):
1131 1122
1132 1123 print('Project[%s]: name = %s, description = %s, project_id = %s' % (self.id,
1133 1124 self.name,
1134 1125 self.description,
1135 1126 self.project_id))
1136 1127
1137 1128 for procUnitConfObj in self.procUnitConfObjDict.values():
1138 1129 print(procUnitConfObj)
1139 1130
1140 1131 def createObjects(self):
1141 1132
1142 1133 for procUnitConfObj in self.procUnitConfObjDict.values():
1143 1134 procUnitConfObj.createObjects()
1144 1135
1145 1136 def __handleError(self, procUnitConfObj, modes=None, stdout=True):
1146 1137
1147 1138 import socket
1148 1139
1149 1140 if modes is None:
1150 1141 modes = self.alarm
1151 1142
1152 1143 if not self.alarm:
1153 1144 modes = []
1154 1145
1155 1146 err = traceback.format_exception(sys.exc_info()[0],
1156 1147 sys.exc_info()[1],
1157 1148 sys.exc_info()[2])
1158 1149
1159 1150 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1160 1151
1161 1152 message = ''.join(err)
1162 1153
1163 1154 if stdout:
1164 1155 sys.stderr.write(message)
1165 1156
1166 1157 subject = 'SChain v%s: Error running %s\n' % (
1167 1158 schainpy.__version__, procUnitConfObj.name)
1168 1159
1169 1160 subtitle = '%s: %s\n' % (
1170 1161 procUnitConfObj.getElementName(), procUnitConfObj.name)
1171 1162 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1172 1163 socket.gethostname())
1173 1164 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1174 1165 subtitle += 'Configuration file: %s\n' % self.filename
1175 1166 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1176 1167
1177 1168 readUnitConfObj = self.getReadUnitObj()
1178 1169 if readUnitConfObj:
1179 1170 subtitle += '\nInput parameters:\n'
1180 1171 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1181 1172 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1182 1173 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1183 1174 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1184 1175 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1185 1176 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1186 1177
1187 1178 a = Alarm(
1188 1179 modes=modes,
1189 1180 email=self.email,
1190 1181 message=message,
1191 1182 subject=subject,
1192 1183 subtitle=subtitle,
1193 1184 filename=self.filename
1194 1185 )
1195 1186
1196 1187 return a
1197 1188
1198 1189 def isPaused(self):
1199 1190 return 0
1200 1191
1201 1192 def isStopped(self):
1202 1193 return 0
1203 1194
1204 1195 def runController(self):
1205 1196 '''
1206 1197 returns 0 when this process has been stopped, 1 otherwise
1207 1198 '''
1208 1199
1209 1200 if self.isPaused():
1210 1201 print('Process suspended')
1211 1202
1212 1203 while True:
1213 1204 time.sleep(0.1)
1214 1205
1215 1206 if not self.isPaused():
1216 1207 break
1217 1208
1218 1209 if self.isStopped():
1219 1210 break
1220 1211
1221 1212 print('Process reinitialized')
1222 1213
1223 1214 if self.isStopped():
1224 1215 print('Process stopped')
1225 1216 return 0
1226 1217
1227 1218 return 1
1228 1219
1229 1220 def setFilename(self, filename):
1230 1221
1231 1222 self.filename = filename
1232 1223
1233 1224 def setProxyCom(self):
1234 1225
1235 1226 if not os.path.exists('/tmp/schain'):
1236 1227 os.mkdir('/tmp/schain')
1237 1228
1238 1229 self.ctx = zmq.Context()
1239 1230 xpub = self.ctx.socket(zmq.XPUB)
1240 1231 xpub.bind('ipc:///tmp/schain/{}_pub'.format(self.id))
1241 1232 xsub = self.ctx.socket(zmq.XSUB)
1242 1233 xsub.bind('ipc:///tmp/schain/{}_sub'.format(self.id))
1243 1234
1244 1235 try:
1245 1236 zmq.proxy(xpub, xsub)
1246 1237 except: # zmq.ContextTerminated:
1247 1238 xpub.close()
1248 1239 xsub.close()
1249 1240
1250 1241 def run(self):
1251 1242
1252 1243 log.success('Starting {}: {}'.format(self.name, self.id), tag='')
1253 1244 self.start_time = time.time()
1254 1245 self.createObjects()
1255 1246 # t = Thread(target=wait, args=(self.ctx, ))
1256 1247 # t.start()
1257 1248 self.setProxyCom()
1258 1249
1259 1250 # Iniciar todos los procesos .start(), monitoreo de procesos. ELiminar lo de abajo
1260 1251
1261 1252 log.success('{} Done (time: {}s)'.format(
1262 1253 self.name,
1263 1254 time.time()-self.start_time))
Show file before | Show file after | Hide comments
@@ -1,1345 +1,1351
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
5 5 '''
6 6
7 7 import copy
8 8 import numpy
9 9 import datetime
10 10 import json
11 11
12 12 from schainpy.utils import log
13 13 from .jroheaderIO import SystemHeader, RadarControllerHeader
14 14
15 15
16 16 def getNumpyDtype(dataTypeCode):
17 17
18 18 if dataTypeCode == 0:
19 19 numpyDtype = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
20 20 elif dataTypeCode == 1:
21 21 numpyDtype = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
22 22 elif dataTypeCode == 2:
23 23 numpyDtype = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
24 24 elif dataTypeCode == 3:
25 25 numpyDtype = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
26 26 elif dataTypeCode == 4:
27 27 numpyDtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
28 28 elif dataTypeCode == 5:
29 29 numpyDtype = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
30 30 else:
31 31 raise ValueError('dataTypeCode was not defined')
32 32
33 33 return numpyDtype
34 34
35 35
36 36 def getDataTypeCode(numpyDtype):
37 37
38 38 if numpyDtype == numpy.dtype([('real', '<i1'), ('imag', '<i1')]):
39 39 datatype = 0
40 40 elif numpyDtype == numpy.dtype([('real', '<i2'), ('imag', '<i2')]):
41 41 datatype = 1
42 42 elif numpyDtype == numpy.dtype([('real', '<i4'), ('imag', '<i4')]):
43 43 datatype = 2
44 44 elif numpyDtype == numpy.dtype([('real', '<i8'), ('imag', '<i8')]):
45 45 datatype = 3
46 46 elif numpyDtype == numpy.dtype([('real', '<f4'), ('imag', '<f4')]):
47 47 datatype = 4
48 48 elif numpyDtype == numpy.dtype([('real', '<f8'), ('imag', '<f8')]):
49 49 datatype = 5
50 50 else:
51 51 datatype = None
52 52
53 53 return datatype
54 54
55 55
56 56 def hildebrand_sekhon(data, navg):
57 57 """
58 58 This method is for the objective determination of the noise level in Doppler spectra. This
59 59 implementation technique is based on the fact that the standard deviation of the spectral
60 60 densities is equal to the mean spectral density for white Gaussian noise
61 61
62 62 Inputs:
63 63 Data : heights
64 64 navg : numbers of averages
65 65
66 66 Return:
67 67 mean : noise's level
68 68 """
69 69
70 70 sortdata = numpy.sort(data, axis=None)
71 71 lenOfData = len(sortdata)
72 72 nums_min = lenOfData*0.2
73 73
74 74 if nums_min <= 5:
75 75
76 76 nums_min = 5
77 77
78 78 sump = 0.
79 79 sumq = 0.
80 80
81 81 j = 0
82 82 cont = 1
83 83
84 84 while((cont == 1)and(j < lenOfData)):
85 85
86 86 sump += sortdata[j]
87 87 sumq += sortdata[j]**2
88 88
89 89 if j > nums_min:
90 90 rtest = float(j)/(j-1) + 1.0/navg
91 91 if ((sumq*j) > (rtest*sump**2)):
92 92 j = j - 1
93 93 sump = sump - sortdata[j]
94 94 sumq = sumq - sortdata[j]**2
95 95 cont = 0
96 96
97 97 j += 1
98 98
99 99 lnoise = sump / j
100 100
101 101 return lnoise
102 102
103 103
104 104 class Beam:
105 105
106 106 def __init__(self):
107 107 self.codeList = []
108 108 self.azimuthList = []
109 109 self.zenithList = []
110 110
111 111
112 112 class GenericData(object):
113 113
114 114 flagNoData = True
115 115
116 116 def copy(self, inputObj=None):
117 117
118 118 if inputObj == None:
119 119 return copy.deepcopy(self)
120 120
121 121 for key in list(inputObj.__dict__.keys()):
122 122
123 123 attribute = inputObj.__dict__[key]
124 124
125 125 # If this attribute is a tuple or list
126 126 if type(inputObj.__dict__[key]) in (tuple, list):
127 127 self.__dict__[key] = attribute[:]
128 128 continue
129 129
130 130 # If this attribute is another object or instance
131 131 if hasattr(attribute, '__dict__'):
132 132 self.__dict__[key] = attribute.copy()
133 133 continue
134 134
135 135 self.__dict__[key] = inputObj.__dict__[key]
136 136
137 137 def deepcopy(self):
138 138
139 139 return copy.deepcopy(self)
140 140
141 141 def isEmpty(self):
142 142
143 143 return self.flagNoData
144 144
145 145
146 146 class JROData(GenericData):
147 147
148 148 # m_BasicHeader = BasicHeader()
149 149 # m_ProcessingHeader = ProcessingHeader()
150 150
151 151 systemHeaderObj = SystemHeader()
152 152 radarControllerHeaderObj = RadarControllerHeader()
153 153 # data = None
154 154 type = None
155 155 datatype = None # dtype but in string
156 156 # dtype = None
157 157 # nChannels = None
158 158 # nHeights = None
159 159 nProfiles = None
160 160 heightList = None
161 161 channelList = None
162 162 flagDiscontinuousBlock = False
163 163 useLocalTime = False
164 164 utctime = None
165 165 timeZone = None
166 166 dstFlag = None
167 167 errorCount = None
168 168 blocksize = None
169 169 # nCode = None
170 170 # nBaud = None
171 171 # code = None
172 172 flagDecodeData = False # asumo q la data no esta decodificada
173 173 flagDeflipData = False # asumo q la data no esta sin flip
174 174 flagShiftFFT = False
175 175 # ippSeconds = None
176 176 # timeInterval = None
177 177 nCohInt = None
178 178 # noise = None
179 179 windowOfFilter = 1
180 180 # Speed of ligth
181 181 C = 3e8
182 182 frequency = 49.92e6
183 183 realtime = False
184 184 beacon_heiIndexList = None
185 185 last_block = None
186 186 blocknow = None
187 187 azimuth = None
188 188 zenith = None
189 189 beam = Beam()
190 190 profileIndex = None
191 191 error = None
192 192 data = None
193 193 nmodes = None
194 194
195 195 def __str__(self):
196 196
197 197 return '{} - {}'.format(self.type, self.getDatatime())
198 198
199 199 def getNoise(self):
200 200
201 201 raise NotImplementedError
202 202
203 203 def getNChannels(self):
204 204
205 205 return len(self.channelList)
206 206
207 207 def getChannelIndexList(self):
208 208
209 209 return list(range(self.nChannels))
210 210
211 211 def getNHeights(self):
212 212
213 213 return len(self.heightList)
214 214
215 215 def getHeiRange(self, extrapoints=0):
216 216
217 217 heis = self.heightList
218 218 # deltah = self.heightList[1] - self.heightList[0]
219 219 #
220 220 # heis.append(self.heightList[-1])
221 221
222 222 return heis
223 223
224 224 def getDeltaH(self):
225 225
226 226 delta = self.heightList[1] - self.heightList[0]
227 227
228 228 return delta
229 229
230 230 def getltctime(self):
231 231
232 232 if self.useLocalTime:
233 233 return self.utctime - self.timeZone * 60
234 234
235 235 return self.utctime
236 236
237 237 def getDatatime(self):
238 238
239 239 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
240 240 return datatimeValue
241 241
242 242 def getTimeRange(self):
243 243
244 244 datatime = []
245 245
246 246 datatime.append(self.ltctime)
247 247 datatime.append(self.ltctime + self.timeInterval + 1)
248 248
249 249 datatime = numpy.array(datatime)
250 250
251 251 return datatime
252 252
253 253 def getFmaxTimeResponse(self):
254 254
255 255 period = (10**-6) * self.getDeltaH() / (0.15)
256 256
257 257 PRF = 1. / (period * self.nCohInt)
258 258
259 259 fmax = PRF
260 260
261 261 return fmax
262 262
263 263 def getFmax(self):
264 264 PRF = 1. / (self.ippSeconds * self.nCohInt)
265 265
266 266 fmax = PRF
267 267 return fmax
268 268
269 269 def getVmax(self):
270 270
271 271 _lambda = self.C / self.frequency
272 272
273 273 vmax = self.getFmax() * _lambda / 2
274 274
275 275 return vmax
276 276
277 277 def get_ippSeconds(self):
278 278 '''
279 279 '''
280 280 return self.radarControllerHeaderObj.ippSeconds
281 281
282 282 def set_ippSeconds(self, ippSeconds):
283 283 '''
284 284 '''
285 285
286 286 self.radarControllerHeaderObj.ippSeconds = ippSeconds
287 287
288 288 return
289 289
290 290 def get_dtype(self):
291 291 '''
292 292 '''
293 293 return getNumpyDtype(self.datatype)
294 294
295 295 def set_dtype(self, numpyDtype):
296 296 '''
297 297 '''
298 298
299 299 self.datatype = getDataTypeCode(numpyDtype)
300 300
301 301 def get_code(self):
302 302 '''
303 303 '''
304 304 return self.radarControllerHeaderObj.code
305 305
306 306 def set_code(self, code):
307 307 '''
308 308 '''
309 309 self.radarControllerHeaderObj.code = code
310 310
311 311 return
312 312
313 313 def get_ncode(self):
314 314 '''
315 315 '''
316 316 return self.radarControllerHeaderObj.nCode
317 317
318 318 def set_ncode(self, nCode):
319 319 '''
320 320 '''
321 321 self.radarControllerHeaderObj.nCode = nCode
322 322
323 323 return
324 324
325 325 def get_nbaud(self):
326 326 '''
327 327 '''
328 328 return self.radarControllerHeaderObj.nBaud
329 329
330 330 def set_nbaud(self, nBaud):
331 331 '''
332 332 '''
333 333 self.radarControllerHeaderObj.nBaud = nBaud
334 334
335 335 return
336 336
337 337 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
338 338 channelIndexList = property(
339 339 getChannelIndexList, "I'm the 'channelIndexList' property.")
340 340 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
341 341 #noise = property(getNoise, "I'm the 'nHeights' property.")
342 342 datatime = property(getDatatime, "I'm the 'datatime' property")
343 343 ltctime = property(getltctime, "I'm the 'ltctime' property")
344 344 ippSeconds = property(get_ippSeconds, set_ippSeconds)
345 345 dtype = property(get_dtype, set_dtype)
346 346 # timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
347 347 code = property(get_code, set_code)
348 348 nCode = property(get_ncode, set_ncode)
349 349 nBaud = property(get_nbaud, set_nbaud)
350 350
351 351
352 352 class Voltage(JROData):
353 353
354 354 # data es un numpy array de 2 dmensiones (canales, alturas)
355 355 data = None
356 356
357 357 def __init__(self):
358 358 '''
359 359 Constructor
360 360 '''
361 361
362 362 self.useLocalTime = True
363 363 self.radarControllerHeaderObj = RadarControllerHeader()
364 364 self.systemHeaderObj = SystemHeader()
365 365 self.type = "Voltage"
366 366 self.data = None
367 367 # self.dtype = None
368 368 # self.nChannels = 0
369 369 # self.nHeights = 0
370 370 self.nProfiles = None
371 self.heightList = Non
371 self.heightList = None
372 372 self.channelList = None
373 373 # self.channelIndexList = None
374 374 self.flagNoData = True
375 375 self.flagDiscontinuousBlock = False
376 376 self.utctime = None
377 377 self.timeZone = None
378 378 self.dstFlag = None
379 379 self.errorCount = None
380 380 self.nCohInt = None
381 381 self.blocksize = None
382 382 self.flagDecodeData = False # asumo q la data no esta decodificada
383 383 self.flagDeflipData = False # asumo q la data no esta sin flip
384 384 self.flagShiftFFT = False
385 385 self.flagDataAsBlock = False # Asumo que la data es leida perfil a perfil
386 386 self.profileIndex = 0
387 387
388 388 def getNoisebyHildebrand(self, channel=None):
389 389 """
390 390 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
391 391
392 392 Return:
393 393 noiselevel
394 394 """
395 395
396 396 if channel != None:
397 397 data = self.data[channel]
398 398 nChannels = 1
399 399 else:
400 400 data = self.data
401 401 nChannels = self.nChannels
402 402
403 403 noise = numpy.zeros(nChannels)
404 404 power = data * numpy.conjugate(data)
405 405
406 406 for thisChannel in range(nChannels):
407 407 if nChannels == 1:
408 408 daux = power[:].real
409 409 else:
410 410 daux = power[thisChannel, :].real
411 411 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
412 412
413 413 return noise
414 414
415 415 def getNoise(self, type=1, channel=None):
416 416
417 417 if type == 1:
418 418 noise = self.getNoisebyHildebrand(channel)
419 419
420 420 return noise
421 421
422 422 def getPower(self, channel=None):
423 423
424 424 if channel != None:
425 425 data = self.data[channel]
426 426 else:
427 427 data = self.data
428 428
429 429 power = data * numpy.conjugate(data)
430 430 powerdB = 10 * numpy.log10(power.real)
431 431 powerdB = numpy.squeeze(powerdB)
432 432
433 433 return powerdB
434 434
435 435 def getTimeInterval(self):
436 436
437 437 timeInterval = self.ippSeconds * self.nCohInt
438 438
439 439 return timeInterval
440 440
441 441 noise = property(getNoise, "I'm the 'nHeights' property.")
442 442 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
443 443
444 444
445 445 class Spectra(JROData):
446 446
447 447 # data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
448 448 data_spc = None
449 449 # data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
450 450 data_cspc = None
451 451 # data dc es un numpy array de 2 dmensiones (canales, alturas)
452 452 data_dc = None
453 453 # data power
454 454 data_pwr = None
455 455 nFFTPoints = None
456 456 # nPairs = None
457 457 pairsList = None
458 458 nIncohInt = None
459 459 wavelength = None # Necesario para cacular el rango de velocidad desde la frecuencia
460 460 nCohInt = None # se requiere para determinar el valor de timeInterval
461 461 ippFactor = None
462 462 profileIndex = 0
463 463 plotting = "spectra"
464 464
465 465 def __init__(self):
466 466 '''
467 467 Constructor
468 468 '''
469 469
470 470 self.useLocalTime = True
471 471 self.radarControllerHeaderObj = RadarControllerHeader()
472 472 self.systemHeaderObj = SystemHeader()
473 473 self.type = "Spectra"
474 474 # self.data = None
475 475 # self.dtype = None
476 476 # self.nChannels = 0
477 477 # self.nHeights = 0
478 478 self.nProfiles = None
479 479 self.heightList = None
480 480 self.channelList = None
481 481 # self.channelIndexList = None
482 482 self.pairsList = None
483 483 self.flagNoData = True
484 484 self.flagDiscontinuousBlock = False
485 485 self.utctime = None
486 486 self.nCohInt = None
487 487 self.nIncohInt = None
488 488 self.blocksize = None
489 489 self.nFFTPoints = None
490 490 self.wavelength = None
491 491 self.flagDecodeData = False # asumo q la data no esta decodificada
492 492 self.flagDeflipData = False # asumo q la data no esta sin flip
493 493 self.flagShiftFFT = False
494 494 self.ippFactor = 1
495 495 #self.noise = None
496 496 self.beacon_heiIndexList = []
497 497 self.noise_estimation = None
498 498
499 499 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
500 500 """
501 501 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
502 502
503 503 Return:
504 504 noiselevel
505 505 """
506 506
507 507 noise = numpy.zeros(self.nChannels)
508 508
509 509 for channel in range(self.nChannels):
510 510 daux = self.data_spc[channel,
511 511 xmin_index:xmax_index, ymin_index:ymax_index]
512 512 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
513 513
514 514 return noise
515 515
516 516 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
517 517
518 518 if self.noise_estimation is not None:
519 519 # this was estimated by getNoise Operation defined in jroproc_spectra.py
520 520 return self.noise_estimation
521 521 else:
522 522 noise = self.getNoisebyHildebrand(
523 523 xmin_index, xmax_index, ymin_index, ymax_index)
524 524 return noise
525 525
526 526 def getFreqRangeTimeResponse(self, extrapoints=0):
527 527
528 528 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints * self.ippFactor)
529 529 freqrange = deltafreq * \
530 530 (numpy.arange(self.nFFTPoints + extrapoints) -
531 531 self.nFFTPoints / 2.) - deltafreq / 2
532 532
533 533 return freqrange
534 534
535 535 def getAcfRange(self, extrapoints=0):
536 536
537 537 deltafreq = 10. / (self.getFmax() / (self.nFFTPoints * self.ippFactor))
538 538 freqrange = deltafreq * \
539 539 (numpy.arange(self.nFFTPoints + extrapoints) -
540 540 self.nFFTPoints / 2.) - deltafreq / 2
541 541
542 542 return freqrange
543 543
544 544 def getFreqRange(self, extrapoints=0):
545 545
546 546 deltafreq = self.getFmax() / (self.nFFTPoints * self.ippFactor)
547 547 freqrange = deltafreq * \
548 548 (numpy.arange(self.nFFTPoints + extrapoints) -
549 549 self.nFFTPoints / 2.) - deltafreq / 2
550 550
551 551 return freqrange
552 552
553 553 def getVelRange(self, extrapoints=0):
554 554
555 555 deltav = self.getVmax() / (self.nFFTPoints * self.ippFactor)
556 556 velrange = deltav * (numpy.arange(self.nFFTPoints +
557 557 extrapoints) - self.nFFTPoints / 2.)
558 558
559 559 if self.nmodes:
560 560 return velrange/self.nmodes
561 561 else:
562 562 return velrange
563 563
564 564 def getNPairs(self):
565 565
566 566 return len(self.pairsList)
567 567
568 568 def getPairsIndexList(self):
569 569
570 570 return list(range(self.nPairs))
571 571
572 572 def getNormFactor(self):
573 573
574 574 pwcode = 1
575 575
576 576 if self.flagDecodeData:
577 577 pwcode = numpy.sum(self.code[0]**2)
578 578 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
579 579 normFactor = self.nProfiles * self.nIncohInt * \
580 580 self.nCohInt * pwcode * self.windowOfFilter
581 581
582 582 return normFactor
583 583
584 584 def getFlagCspc(self):
585 585
586 586 if self.data_cspc is None:
587 587 return True
588 588
589 589 return False
590 590
591 591 def getFlagDc(self):
592 592
593 593 if self.data_dc is None:
594 594 return True
595 595
596 596 return False
597 597
598 598 def getTimeInterval(self):
599 599
600 600 timeInterval = self.ippSeconds * self.nCohInt * \
601 601 self.nIncohInt * self.nProfiles * self.ippFactor
602 602
603 603 return timeInterval
604 604
605 605 def getPower(self):
606 606
607 607 factor = self.normFactor
608 608 z = self.data_spc / factor
609 609 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
610 610 avg = numpy.average(z, axis=1)
611 611
612 612 return 10 * numpy.log10(avg)
613 613
614 614 def getCoherence(self, pairsList=None, phase=False):
615 615
616 616 z = []
617 617 if pairsList is None:
618 618 pairsIndexList = self.pairsIndexList
619 619 else:
620 620 pairsIndexList = []
621 621 for pair in pairsList:
622 622 if pair not in self.pairsList:
623 623 raise ValueError("Pair %s is not in dataOut.pairsList" % (
624 624 pair))
625 625 pairsIndexList.append(self.pairsList.index(pair))
626 626 for i in range(len(pairsIndexList)):
627 627 pair = self.pairsList[pairsIndexList[i]]
628 628 ccf = numpy.average(
629 629 self.data_cspc[pairsIndexList[i], :, :], axis=0)
630 630 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
631 631 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
632 632 avgcoherenceComplex = ccf / numpy.sqrt(powa * powb)
633 633 if phase:
634 634 data = numpy.arctan2(avgcoherenceComplex.imag,
635 635 avgcoherenceComplex.real) * 180 / numpy.pi
636 636 else:
637 637 data = numpy.abs(avgcoherenceComplex)
638 638
639 639 z.append(data)
640 640
641 641 return numpy.array(z)
642 642
643 643 def setValue(self, value):
644 644
645 645 print("This property should not be initialized")
646 646
647 647 return
648 648
649 649 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
650 650 pairsIndexList = property(
651 651 getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
652 652 normFactor = property(getNormFactor, setValue,
653 653 "I'm the 'getNormFactor' property.")
654 654 flag_cspc = property(getFlagCspc, setValue)
655 655 flag_dc = property(getFlagDc, setValue)
656 656 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
657 657 timeInterval = property(getTimeInterval, setValue,
658 658 "I'm the 'timeInterval' property")
659 659
660 660
661 661 class SpectraHeis(Spectra):
662 662
663 663 data_spc = None
664 664 data_cspc = None
665 665 data_dc = None
666 666 nFFTPoints = None
667 667 # nPairs = None
668 668 pairsList = None
669 669 nCohInt = None
670 670 nIncohInt = None
671 671
672 672 def __init__(self):
673 673
674 674 self.radarControllerHeaderObj = RadarControllerHeader()
675 675
676 676 self.systemHeaderObj = SystemHeader()
677 677
678 678 self.type = "SpectraHeis"
679 679
680 680 # self.dtype = None
681 681
682 682 # self.nChannels = 0
683 683
684 684 # self.nHeights = 0
685 685
686 686 self.nProfiles = None
687 687
688 688 self.heightList = None
689 689
690 690 self.channelList = None
691 691
692 692 # self.channelIndexList = None
693 693
694 694 self.flagNoData = True
695 695
696 696 self.flagDiscontinuousBlock = False
697 697
698 698 # self.nPairs = 0
699 699
700 700 self.utctime = None
701 701
702 702 self.blocksize = None
703 703
704 704 self.profileIndex = 0
705 705
706 706 self.nCohInt = 1
707 707
708 708 self.nIncohInt = 1
709 709
710 710 def getNormFactor(self):
711 711 pwcode = 1
712 712 if self.flagDecodeData:
713 713 pwcode = numpy.sum(self.code[0]**2)
714 714
715 715 normFactor = self.nIncohInt * self.nCohInt * pwcode
716 716
717 717 return normFactor
718 718
719 719 def getTimeInterval(self):
720 720
721 721 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
722 722
723 723 return timeInterval
724 724
725 725 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
726 726 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
727 727
728 728
729 729 class Fits(JROData):
730 730
731 731 heightList = None
732 732 channelList = None
733 733 flagNoData = True
734 734 flagDiscontinuousBlock = False
735 735 useLocalTime = False
736 736 utctime = None
737 737 timeZone = None
738 738 # ippSeconds = None
739 739 # timeInterval = None
740 740 nCohInt = None
741 741 nIncohInt = None
742 742 noise = None
743 743 windowOfFilter = 1
744 744 # Speed of ligth
745 745 C = 3e8
746 746 frequency = 49.92e6
747 747 realtime = False
748 748
749 749 def __init__(self):
750 750
751 751 self.type = "Fits"
752 752
753 753 self.nProfiles = None
754 754
755 755 self.heightList = None
756 756
757 757 self.channelList = None
758 758
759 759 # self.channelIndexList = None
760 760
761 761 self.flagNoData = True
762 762
763 763 self.utctime = None
764 764
765 765 self.nCohInt = 1
766 766
767 767 self.nIncohInt = 1
768 768
769 769 self.useLocalTime = True
770 770
771 771 self.profileIndex = 0
772 772
773 773 # self.utctime = None
774 774 # self.timeZone = None
775 775 # self.ltctime = None
776 776 # self.timeInterval = None
777 777 # self.header = None
778 778 # self.data_header = None
779 779 # self.data = None
780 780 # self.datatime = None
781 781 # self.flagNoData = False
782 782 # self.expName = ''
783 783 # self.nChannels = None
784 784 # self.nSamples = None
785 785 # self.dataBlocksPerFile = None
786 786 # self.comments = ''
787 787 #
788 788
789 789 def getltctime(self):
790 790
791 791 if self.useLocalTime:
792 792 return self.utctime - self.timeZone * 60
793 793
794 794 return self.utctime
795 795
796 796 def getDatatime(self):
797 797
798 798 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
799 799 return datatime
800 800
801 801 def getTimeRange(self):
802 802
803 803 datatime = []
804 804
805 805 datatime.append(self.ltctime)
806 806 datatime.append(self.ltctime + self.timeInterval)
807 807
808 808 datatime = numpy.array(datatime)
809 809
810 810 return datatime
811 811
812 812 def getHeiRange(self):
813 813
814 814 heis = self.heightList
815 815
816 816 return heis
817 817
818 818 def getNHeights(self):
819 819
820 820 return len(self.heightList)
821 821
822 822 def getNChannels(self):
823 823
824 824 return len(self.channelList)
825 825
826 826 def getChannelIndexList(self):
827 827
828 828 return list(range(self.nChannels))
829 829
830 830 def getNoise(self, type=1):
831 831
832 832 #noise = numpy.zeros(self.nChannels)
833 833
834 834 if type == 1:
835 835 noise = self.getNoisebyHildebrand()
836 836
837 837 if type == 2:
838 838 noise = self.getNoisebySort()
839 839
840 840 if type == 3:
841 841 noise = self.getNoisebyWindow()
842 842
843 843 return noise
844 844
845 845 def getTimeInterval(self):
846 846
847 847 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
848 848
849 849 return timeInterval
850 850
851 def get_ippSeconds(self):
852 '''
853 '''
854 return self.ipp_sec
855
856
851 857 datatime = property(getDatatime, "I'm the 'datatime' property")
852 858 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
853 859 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
854 860 channelIndexList = property(
855 861 getChannelIndexList, "I'm the 'channelIndexList' property.")
856 862 noise = property(getNoise, "I'm the 'nHeights' property.")
857 863
858 864 ltctime = property(getltctime, "I'm the 'ltctime' property")
859 865 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
860
866 ippSeconds = property(get_ippSeconds, '')
861 867
862 868 class Correlation(JROData):
863 869
864 870 noise = None
865 871 SNR = None
866 872 #--------------------------------------------------
867 873 mode = None
868 874 split = False
869 875 data_cf = None
870 876 lags = None
871 877 lagRange = None
872 878 pairsList = None
873 879 normFactor = None
874 880 #--------------------------------------------------
875 881 # calculateVelocity = None
876 882 nLags = None
877 883 nPairs = None
878 884 nAvg = None
879 885
880 886 def __init__(self):
881 887 '''
882 888 Constructor
883 889 '''
884 890 self.radarControllerHeaderObj = RadarControllerHeader()
885 891
886 892 self.systemHeaderObj = SystemHeader()
887 893
888 894 self.type = "Correlation"
889 895
890 896 self.data = None
891 897
892 898 self.dtype = None
893 899
894 900 self.nProfiles = None
895 901
896 902 self.heightList = None
897 903
898 904 self.channelList = None
899 905
900 906 self.flagNoData = True
901 907
902 908 self.flagDiscontinuousBlock = False
903 909
904 910 self.utctime = None
905 911
906 912 self.timeZone = None
907 913
908 914 self.dstFlag = None
909 915
910 916 self.errorCount = None
911 917
912 918 self.blocksize = None
913 919
914 920 self.flagDecodeData = False # asumo q la data no esta decodificada
915 921
916 922 self.flagDeflipData = False # asumo q la data no esta sin flip
917 923
918 924 self.pairsList = None
919 925
920 926 self.nPoints = None
921 927
922 928 def getPairsList(self):
923 929
924 930 return self.pairsList
925 931
926 932 def getNoise(self, mode=2):
927 933
928 934 indR = numpy.where(self.lagR == 0)[0][0]
929 935 indT = numpy.where(self.lagT == 0)[0][0]
930 936
931 937 jspectra0 = self.data_corr[:, :, indR, :]
932 938 jspectra = copy.copy(jspectra0)
933 939
934 940 num_chan = jspectra.shape[0]
935 941 num_hei = jspectra.shape[2]
936 942
937 943 freq_dc = jspectra.shape[1] / 2
938 944 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
939 945
940 946 if ind_vel[0] < 0:
941 947 ind_vel[list(range(0, 1))] = ind_vel[list(
942 948 range(0, 1))] + self.num_prof
943 949
944 950 if mode == 1:
945 951 jspectra[:, freq_dc, :] = (
946 952 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
947 953
948 954 if mode == 2:
949 955
950 956 vel = numpy.array([-2, -1, 1, 2])
951 957 xx = numpy.zeros([4, 4])
952 958
953 959 for fil in range(4):
954 960 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
955 961
956 962 xx_inv = numpy.linalg.inv(xx)
957 963 xx_aux = xx_inv[0, :]
958 964
959 965 for ich in range(num_chan):
960 966 yy = jspectra[ich, ind_vel, :]
961 967 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
962 968
963 969 junkid = jspectra[ich, freq_dc, :] <= 0
964 970 cjunkid = sum(junkid)
965 971
966 972 if cjunkid.any():
967 973 jspectra[ich, freq_dc, junkid.nonzero()] = (
968 974 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
969 975
970 976 noise = jspectra0[:, freq_dc, :] - jspectra[:, freq_dc, :]
971 977
972 978 return noise
973 979
974 980 def getTimeInterval(self):
975 981
976 982 timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
977 983
978 984 return timeInterval
979 985
980 986 def splitFunctions(self):
981 987
982 988 pairsList = self.pairsList
983 989 ccf_pairs = []
984 990 acf_pairs = []
985 991 ccf_ind = []
986 992 acf_ind = []
987 993 for l in range(len(pairsList)):
988 994 chan0 = pairsList[l][0]
989 995 chan1 = pairsList[l][1]
990 996
991 997 # Obteniendo pares de Autocorrelacion
992 998 if chan0 == chan1:
993 999 acf_pairs.append(chan0)
994 1000 acf_ind.append(l)
995 1001 else:
996 1002 ccf_pairs.append(pairsList[l])
997 1003 ccf_ind.append(l)
998 1004
999 1005 data_acf = self.data_cf[acf_ind]
1000 1006 data_ccf = self.data_cf[ccf_ind]
1001 1007
1002 1008 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
1003 1009
1004 1010 def getNormFactor(self):
1005 1011 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
1006 1012 acf_pairs = numpy.array(acf_pairs)
1007 1013 normFactor = numpy.zeros((self.nPairs, self.nHeights))
1008 1014
1009 1015 for p in range(self.nPairs):
1010 1016 pair = self.pairsList[p]
1011 1017
1012 1018 ch0 = pair[0]
1013 1019 ch1 = pair[1]
1014 1020
1015 1021 ch0_max = numpy.max(data_acf[acf_pairs == ch0, :, :], axis=1)
1016 1022 ch1_max = numpy.max(data_acf[acf_pairs == ch1, :, :], axis=1)
1017 1023 normFactor[p, :] = numpy.sqrt(ch0_max * ch1_max)
1018 1024
1019 1025 return normFactor
1020 1026
1021 1027 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
1022 1028 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
1023 1029
1024 1030
1025 1031 class Parameters(Spectra):
1026 1032
1027 1033 experimentInfo = None # Information about the experiment
1028 1034 # Information from previous data
1029 1035 inputUnit = None # Type of data to be processed
1030 1036 operation = None # Type of operation to parametrize
1031 1037 # normFactor = None #Normalization Factor
1032 1038 groupList = None # List of Pairs, Groups, etc
1033 1039 # Parameters
1034 1040 data_param = None # Parameters obtained
1035 1041 data_pre = None # Data Pre Parametrization
1036 1042 data_SNR = None # Signal to Noise Ratio
1037 1043 # heightRange = None #Heights
1038 1044 abscissaList = None # Abscissa, can be velocities, lags or time
1039 1045 # noise = None #Noise Potency
1040 1046 utctimeInit = None # Initial UTC time
1041 1047 paramInterval = None # Time interval to calculate Parameters in seconds
1042 1048 useLocalTime = True
1043 1049 # Fitting
1044 1050 data_error = None # Error of the estimation
1045 1051 constants = None
1046 1052 library = None
1047 1053 # Output signal
1048 1054 outputInterval = None # Time interval to calculate output signal in seconds
1049 1055 data_output = None # Out signal
1050 1056 nAvg = None
1051 1057 noise_estimation = None
1052 1058 GauSPC = None # Fit gaussian SPC
1053 1059
1054 1060 def __init__(self):
1055 1061 '''
1056 1062 Constructor
1057 1063 '''
1058 1064 self.radarControllerHeaderObj = RadarControllerHeader()
1059 1065
1060 1066 self.systemHeaderObj = SystemHeader()
1061 1067
1062 1068 self.type = "Parameters"
1063 1069
1064 1070 def getTimeRange1(self, interval):
1065 1071
1066 1072 datatime = []
1067 1073
1068 1074 if self.useLocalTime:
1069 1075 time1 = self.utctimeInit - self.timeZone * 60
1070 1076 else:
1071 1077 time1 = self.utctimeInit
1072 1078
1073 1079 datatime.append(time1)
1074 1080 datatime.append(time1 + interval)
1075 1081 datatime = numpy.array(datatime)
1076 1082
1077 1083 return datatime
1078 1084
1079 1085 def getTimeInterval(self):
1080 1086
1081 1087 if hasattr(self, 'timeInterval1'):
1082 1088 return self.timeInterval1
1083 1089 else:
1084 1090 return self.paramInterval
1085 1091
1086 1092 def setValue(self, value):
1087 1093
1088 1094 print("This property should not be initialized")
1089 1095
1090 1096 return
1091 1097
1092 1098 def getNoise(self):
1093 1099
1094 1100 return self.spc_noise
1095 1101
1096 1102 timeInterval = property(getTimeInterval)
1097 1103 noise = property(getNoise, setValue, "I'm the 'Noise' property.")
1098 1104
1099 1105
1100 1106 class PlotterData(object):
1101 1107 '''
1102 1108 Object to hold data to be plotted
1103 1109 '''
1104 1110
1105 1111 MAXNUMX = 100
1106 1112 MAXNUMY = 100
1107 1113
1108 1114 def __init__(self, code, throttle_value, exp_code, buffering=True):
1109 1115
1110 1116 self.throttle = throttle_value
1111 1117 self.exp_code = exp_code
1112 1118 self.buffering = buffering
1113 1119 self.ready = False
1114 1120 self.localtime = False
1115 1121 self.data = {}
1116 1122 self.meta = {}
1117 1123 self.__times = []
1118 1124 self.__heights = []
1119 1125
1120 1126 if 'snr' in code:
1121 1127 self.plottypes = ['snr']
1122 1128 elif code == 'spc':
1123 1129 self.plottypes = ['spc', 'noise', 'rti']
1124 1130 elif code == 'rti':
1125 1131 self.plottypes = ['noise', 'rti']
1126 1132 else:
1127 1133 self.plottypes = [code]
1128 1134
1129 1135 for plot in self.plottypes:
1130 1136 self.data[plot] = {}
1131 1137
1132 1138 def __str__(self):
1133 1139 dum = ['{}{}'.format(key, self.shape(key)) for key in self.data]
1134 1140 return 'Data[{}][{}]'.format(';'.join(dum), len(self.__times))
1135 1141
1136 1142 def __len__(self):
1137 1143 return len(self.__times)
1138 1144
1139 1145 def __getitem__(self, key):
1140 1146
1141 1147 if key not in self.data:
1142 1148 raise KeyError(log.error('Missing key: {}'.format(key)))
1143 1149 if 'spc' in key or not self.buffering:
1144 1150 ret = self.data[key]
1145 1151 else:
1146 1152 ret = numpy.array([self.data[key][x] for x in self.times])
1147 1153 if ret.ndim > 1:
1148 1154 ret = numpy.swapaxes(ret, 0, 1)
1149 1155 return ret
1150 1156
1151 1157 def __contains__(self, key):
1152 1158 return key in self.data
1153 1159
1154 1160 def setup(self):
1155 1161 '''
1156 1162 Configure object
1157 1163 '''
1158 1164
1159 1165 self.type = ''
1160 1166 self.ready = False
1161 1167 self.data = {}
1162 1168 self.__times = []
1163 1169 self.__heights = []
1164 1170 self.__all_heights = set()
1165 1171 for plot in self.plottypes:
1166 1172 if 'snr' in plot:
1167 1173 plot = 'snr'
1168 1174 self.data[plot] = {}
1169 1175
1170 1176 if 'spc' in self.data or 'rti' in self.data:
1171 1177 self.data['noise'] = {}
1172 1178 if 'noise' not in self.plottypes:
1173 1179 self.plottypes.append('noise')
1174 1180
1175 1181 def shape(self, key):
1176 1182 '''
1177 1183 Get the shape of the one-element data for the given key
1178 1184 '''
1179 1185
1180 1186 if len(self.data[key]):
1181 1187 if 'spc' in key or not self.buffering:
1182 1188 return self.data[key].shape
1183 1189 return self.data[key][self.__times[0]].shape
1184 1190 return (0,)
1185 1191
1186 1192 def update(self, dataOut, tm):
1187 1193 '''
1188 1194 Update data object with new dataOut
1189 1195 '''
1190 1196
1191 1197 if tm in self.__times:
1192 1198 return
1193 1199
1194 1200 self.type = dataOut.type
1195 1201 self.parameters = getattr(dataOut, 'parameters', [])
1196 1202 if hasattr(dataOut, 'pairsList'):
1197 1203 self.pairs = dataOut.pairsList
1198 1204 if hasattr(dataOut, 'meta'):
1199 1205 self.meta = dataOut.meta
1200 1206 self.channels = dataOut.channelList
1201 1207 self.interval = dataOut.getTimeInterval()
1202 1208 self.localtime = dataOut.useLocalTime
1203 1209 if 'spc' in self.plottypes or 'cspc' in self.plottypes:
1204 1210 self.xrange = (dataOut.getFreqRange(1)/1000.,
1205 1211 dataOut.getAcfRange(1), dataOut.getVelRange(1))
1206 1212 self.__heights.append(dataOut.heightList)
1207 1213 self.__all_heights.update(dataOut.heightList)
1208 1214 self.__times.append(tm)
1209 1215
1210 1216 for plot in self.plottypes:
1211 1217 if plot == 'spc':
1212 1218 z = dataOut.data_spc/dataOut.normFactor
1213 1219 buffer = 10*numpy.log10(z)
1214 1220 if plot == 'cspc':
1215 1221 buffer = dataOut.data_cspc
1216 1222 if plot == 'noise':
1217 1223 buffer = 10*numpy.log10(dataOut.getNoise()/dataOut.normFactor)
1218 1224 if plot == 'rti':
1219 1225 buffer = dataOut.getPower()
1220 1226 if plot == 'snr_db':
1221 1227 buffer = dataOut.data_SNR
1222 1228 if plot == 'snr':
1223 1229 buffer = 10*numpy.log10(dataOut.data_SNR)
1224 1230 if plot == 'dop':
1225 1231 buffer = 10*numpy.log10(dataOut.data_DOP)
1226 1232 if plot == 'mean':
1227 1233 buffer = dataOut.data_MEAN
1228 1234 if plot == 'std':
1229 1235 buffer = dataOut.data_STD
1230 1236 if plot == 'coh':
1231 1237 buffer = dataOut.getCoherence()
1232 1238 if plot == 'phase':
1233 1239 buffer = dataOut.getCoherence(phase=True)
1234 1240 if plot == 'output':
1235 1241 buffer = dataOut.data_output
1236 1242 if plot == 'param':
1237 1243 buffer = dataOut.data_param
1238 1244
1239 1245 if 'spc' in plot:
1240 1246 self.data[plot] = buffer
1241 1247 else:
1242 1248 if self.buffering:
1243 1249 self.data[plot][tm] = buffer
1244 1250 else:
1245 1251 self.data[plot] = buffer
1246 1252
1247 1253 def normalize_heights(self):
1248 1254 '''
1249 1255 Ensure same-dimension of the data for different heighList
1250 1256 '''
1251 1257
1252 1258 H = numpy.array(list(self.__all_heights))
1253 1259 H.sort()
1254 1260 for key in self.data:
1255 1261 shape = self.shape(key)[:-1] + H.shape
1256 1262 for tm, obj in list(self.data[key].items()):
1257 1263 h = self.__heights[self.__times.index(tm)]
1258 1264 if H.size == h.size:
1259 1265 continue
1260 1266 index = numpy.where(numpy.in1d(H, h))[0]
1261 1267 dummy = numpy.zeros(shape) + numpy.nan
1262 1268 if len(shape) == 2:
1263 1269 dummy[:, index] = obj
1264 1270 else:
1265 1271 dummy[index] = obj
1266 1272 self.data[key][tm] = dummy
1267 1273
1268 1274 self.__heights = [H for tm in self.__times]
1269 1275
1270 1276 def jsonify(self, decimate=False):
1271 1277 '''
1272 1278 Convert data to json
1273 1279 '''
1274 1280
1275 1281 data = {}
1276 1282 tm = self.times[-1]
1277 1283 dy = int(self.heights.size/self.MAXNUMY) + 1
1278 1284 for key in self.data:
1279 1285 if key in ('spc', 'cspc') or not self.buffering:
1280 1286 dx = int(self.data[key].shape[1]/self.MAXNUMX) + 1
1281 1287 data[key] = self.roundFloats(
1282 1288 self.data[key][::, ::dx, ::dy].tolist())
1283 1289 else:
1284 1290 data[key] = self.roundFloats(self.data[key][tm].tolist())
1285 1291
1286 1292 ret = {'data': data}
1287 1293 ret['exp_code'] = self.exp_code
1288 1294 ret['time'] = float(tm)
1289 1295 ret['interval'] = float(self.interval)
1290 1296 ret['localtime'] = self.localtime
1291 1297 ret['yrange'] = self.roundFloats(self.heights[::dy].tolist())
1292 1298 if 'spc' in self.data or 'cspc' in self.data:
1293 1299 ret['xrange'] = self.roundFloats(self.xrange[2][::dx].tolist())
1294 1300 else:
1295 1301 ret['xrange'] = []
1296 1302 if hasattr(self, 'pairs'):
1297 1303 ret['pairs'] = [(int(p[0]), int(p[1])) for p in self.pairs]
1298 1304 else:
1299 1305 ret['pairs'] = []
1300 1306
1301 1307 for key, value in list(self.meta.items()):
1302 1308 ret[key] = value
1303 1309
1304 1310 return json.dumps(ret)
1305 1311
1306 1312 @property
1307 1313 def times(self):
1308 1314 '''
1309 1315 Return the list of times of the current data
1310 1316 '''
1311 1317
1312 1318 ret = numpy.array(self.__times)
1313 1319 ret.sort()
1314 1320 return ret
1315 1321
1316 1322 @property
1317 1323 def min_time(self):
1318 1324 '''
1319 1325 Return the minimun time value
1320 1326 '''
1321 1327
1322 1328 return self.times[0]
1323 1329
1324 1330 @property
1325 1331 def max_time(self):
1326 1332 '''
1327 1333 Return the maximun time value
1328 1334 '''
1329 1335
1330 1336 return self.times[-1]
1331 1337
1332 1338 @property
1333 1339 def heights(self):
1334 1340 '''
1335 1341 Return the list of heights of the current data
1336 1342 '''
1337 1343
1338 1344 return numpy.array(self.__heights[-1])
1339 1345
1340 1346 @staticmethod
1341 1347 def roundFloats(obj):
1342 1348 if isinstance(obj, list):
1343 1349 return list(map(PlotterData.roundFloats, obj))
1344 1350 elif isinstance(obj, float):
1345 1351 return round(obj, 2)
Show file before | Show file after | Hide comments
@@ -1,329 +1,345
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from .figure import Figure, isRealtime
11 11 from .plotting_codes import *
12 from schainpy.model.proc.jroproc_base import MPDecorator
12 13
14
15 @MPDecorator
13 16 class SpectraHeisScope_(Figure):
14 17
15 18
16 19 isConfig = None
17 20 __nsubplots = None
18 21
19 22 WIDTHPROF = None
20 23 HEIGHTPROF = None
21 24 PREFIX = 'spc'
22 25
23 def __init__(self, **kwargs):
26 def __init__(self):#, **kwargs):
24 27
25 Figure.__init__(self, **kwargs)
28 Figure.__init__(self)#, **kwargs)
26 29 self.isConfig = False
27 30 self.__nsubplots = 1
28 31
29 32 self.WIDTH = 230
30 33 self.HEIGHT = 250
31 34 self.WIDTHPROF = 120
32 35 self.HEIGHTPROF = 0
33 36 self.counter_imagwr = 0
34 37
35 38 self.PLOT_CODE = SPEC_CODE
36 39
37 40 def getSubplots(self):
38 41
39 42 ncol = int(numpy.sqrt(self.nplots)+0.9)
40 43 nrow = int(self.nplots*1./ncol + 0.9)
41 44
42 45 return nrow, ncol
43 46
44 47 def setup(self, id, nplots, wintitle, show):
45 48
46 49 showprofile = False
47 50 self.__showprofile = showprofile
48 51 self.nplots = nplots
49 52
50 53 ncolspan = 1
51 54 colspan = 1
52 55 if showprofile:
53 56 ncolspan = 3
54 57 colspan = 2
55 58 self.__nsubplots = 2
56 59
57 60 self.createFigure(id = id,
58 61 wintitle = wintitle,
59 62 widthplot = self.WIDTH + self.WIDTHPROF,
60 63 heightplot = self.HEIGHT + self.HEIGHTPROF,
61 64 show = show)
62 65
63 66 nrow, ncol = self.getSubplots()
64 67
65 68 counter = 0
66 69 for y in range(nrow):
67 70 for x in range(ncol):
68 71
69 72 if counter >= self.nplots:
70 73 break
71 74
72 75 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
73 76
74 77 if showprofile:
75 78 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
76 79
77 80 counter += 1
78 81
79 82
80 83 def run(self, dataOut, id, wintitle="", channelList=None,
81 84 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
82 85 figpath='./', figfile=None, ftp=False, wr_period=1, show=True,
83 86 server=None, folder=None, username=None, password=None,
84 87 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
85 88
86 89 """
87 90
88 91 Input:
89 92 dataOut :
90 93 id :
91 94 wintitle :
92 95 channelList :
93 96 xmin : None,
94 97 xmax : None,
95 98 ymin : None,
96 99 ymax : None,
97 100 """
98 101
102 if dataOut.flagNoData:
103 return dataOut
104
99 105 if dataOut.realtime:
100 106 if not(isRealtime(utcdatatime = dataOut.utctime)):
101 107 print('Skipping this plot function')
102 108 return
103 109
104 110 if channelList == None:
105 111 channelIndexList = dataOut.channelIndexList
106 112 else:
107 113 channelIndexList = []
108 114 for channel in channelList:
109 115 if channel not in dataOut.channelList:
110 116 raise ValueError("Channel %d is not in dataOut.channelList")
111 117 channelIndexList.append(dataOut.channelList.index(channel))
112 118
113 119 # x = dataOut.heightList
114 120 c = 3E8
115 121 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
116 122 #deberia cambiar para el caso de 1Mhz y 100KHz
117 123 x = numpy.arange(-1*dataOut.nHeights/2.,dataOut.nHeights/2.)*(c/(2*deltaHeight*dataOut.nHeights*1000))
118 124 #para 1Mhz descomentar la siguiente linea
119 125 #x= x/(10000.0)
120 126 # y = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
121 127 # y = y.real
122 128 factor = dataOut.normFactor
123 129 data = dataOut.data_spc / factor
124 130 datadB = 10.*numpy.log10(data)
125 131 y = datadB
126 132
127 133 #thisDatetime = dataOut.datatime
128 134 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
129 135 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
130 136 xlabel = ""
131 137 #para 1Mhz descomentar la siguiente linea
132 138 #xlabel = "Frequency x 10000"
133 139 ylabel = "Intensity (dB)"
134 140
135 141 if not self.isConfig:
136 142 nplots = len(channelIndexList)
137 143
138 144 self.setup(id=id,
139 145 nplots=nplots,
140 146 wintitle=wintitle,
141 147 show=show)
142 148
143 149 if xmin == None: xmin = numpy.nanmin(x)
144 150 if xmax == None: xmax = numpy.nanmax(x)
145 151 if ymin == None: ymin = numpy.nanmin(y)
146 152 if ymax == None: ymax = numpy.nanmax(y)
147 153
148 154 self.FTP_WEI = ftp_wei
149 155 self.EXP_CODE = exp_code
150 156 self.SUB_EXP_CODE = sub_exp_code
151 157 self.PLOT_POS = plot_pos
152 158
153 159 self.isConfig = True
154 160
155 161 self.setWinTitle(title)
156 162
157 163 for i in range(len(self.axesList)):
158 164 ychannel = y[i,:]
159 165 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
160 166 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[channelIndexList[i]], numpy.max(ychannel), str_datetime)
161 167 axes = self.axesList[i]
162 168 axes.pline(x, ychannel,
163 169 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
164 170 xlabel=xlabel, ylabel=ylabel, title=title, grid='both')
165 171
166 172
167 173 self.draw()
168 174
169 175 self.save(figpath=figpath,
170 176 figfile=figfile,
171 177 save=save,
172 178 ftp=ftp,
173 179 wr_period=wr_period,
174 180 thisDatetime=thisDatetime)
175 181
182 return dataOut
183
184 @MPDecorator
176 185 class RTIfromSpectraHeis_(Figure):
177 186
178 187 isConfig = None
179 188 __nsubplots = None
180 189
181 190 PREFIX = 'rtinoise'
182 191
183 def __init__(self, **kwargs):
184 Figure.__init__(self, **kwargs)
192 def __init__(self):#, **kwargs):
193 Figure.__init__(self)#, **kwargs)
185 194 self.timerange = 24*60*60
186 195 self.isConfig = False
187 196 self.__nsubplots = 1
188 197
189 198 self.WIDTH = 820
190 199 self.HEIGHT = 200
191 200 self.WIDTHPROF = 120
192 201 self.HEIGHTPROF = 0
193 202 self.counter_imagwr = 0
194 203 self.xdata = None
195 204 self.ydata = None
196 205 self.figfile = None
197 206
198 207 self.PLOT_CODE = RTI_CODE
199 208
200 209 def getSubplots(self):
201 210
202 211 ncol = 1
203 212 nrow = 1
204 213
205 214 return nrow, ncol
206 215
207 216 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
208 217
209 218 self.__showprofile = showprofile
210 219 self.nplots = nplots
211 220
212 221 ncolspan = 7
213 222 colspan = 6
214 223 self.__nsubplots = 2
215 224
216 225 self.createFigure(id = id,
217 226 wintitle = wintitle,
218 227 widthplot = self.WIDTH+self.WIDTHPROF,
219 228 heightplot = self.HEIGHT+self.HEIGHTPROF,
220 229 show = show)
221 230
222 231 nrow, ncol = self.getSubplots()
223 232
224 233 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
225 234
226 235
227 236 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
228 237 xmin=None, xmax=None, ymin=None, ymax=None,
229 238 timerange=None,
230 239 save=False, figpath='./', figfile=None, ftp=False, wr_period=1, show=True,
231 240 server=None, folder=None, username=None, password=None,
232 241 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
233 242
243 if dataOut.flagNoData:
244 return dataOut
245
246
234 247 if channelList == None:
235 248 channelIndexList = dataOut.channelIndexList
236 249 channelList = dataOut.channelList
237 250 else:
238 251 channelIndexList = []
239 252 for channel in channelList:
240 253 if channel not in dataOut.channelList:
241 254 raise ValueError("Channel %d is not in dataOut.channelList")
242 255 channelIndexList.append(dataOut.channelList.index(channel))
243 256
244 257 if timerange != None:
245 258 self.timerange = timerange
246 259
247 260 x = dataOut.getTimeRange()
248 261 y = dataOut.getHeiRange()
249 262
250 263 factor = dataOut.normFactor
251 264 data = dataOut.data_spc / factor
252 265 data = numpy.average(data,axis=1)
253 266 datadB = 10*numpy.log10(data)
254 267
255 268 # factor = dataOut.normFactor
256 269 # noise = dataOut.getNoise()/factor
257 270 # noisedB = 10*numpy.log10(noise)
258 271
259 272 #thisDatetime = dataOut.datatime
260 273 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
261 274 title = wintitle + " RTI: %s" %(thisDatetime.strftime("%d-%b-%Y"))
262 275 xlabel = "Local Time"
263 276 ylabel = "Intensity (dB)"
264 277
265 278 if not self.isConfig:
266 279
267 280 nplots = 1
268 281
269 282 self.setup(id=id,
270 283 nplots=nplots,
271 284 wintitle=wintitle,
272 285 showprofile=showprofile,
273 286 show=show)
274 287
275 288 self.tmin, self.tmax = self.getTimeLim(x, xmin, xmax)
276 289
277 290 if ymin == None: ymin = numpy.nanmin(datadB)
278 291 if ymax == None: ymax = numpy.nanmax(datadB)
279 292
280 293 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
281 294 self.isConfig = True
282 295 self.figfile = figfile
283 296 self.xdata = numpy.array([])
284 297 self.ydata = numpy.array([])
285 298
286 299 self.FTP_WEI = ftp_wei
287 300 self.EXP_CODE = exp_code
288 301 self.SUB_EXP_CODE = sub_exp_code
289 302 self.PLOT_POS = plot_pos
290 303
291 304 self.setWinTitle(title)
292 305
293 306
294 307 # title = "RTI %s" %(thisDatetime.strftime("%d-%b-%Y"))
295 308 title = "RTI - %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
296 309
297 310 legendlabels = ["channel %d"%idchannel for idchannel in channelList]
298 311 axes = self.axesList[0]
299 312
300 313 self.xdata = numpy.hstack((self.xdata, x[0:1]))
301 314
302 315 if len(self.ydata)==0:
303 316 self.ydata = datadB[channelIndexList].reshape(-1,1)
304 317 else:
305 318 self.ydata = numpy.hstack((self.ydata, datadB[channelIndexList].reshape(-1,1)))
306 319
307 320
308 321 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
309 322 xmin=self.tmin, xmax=self.tmax, ymin=ymin, ymax=ymax,
310 323 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='.', markersize=8, linestyle="solid", grid='both',
311 324 XAxisAsTime=True
312 325 )
313 326
314 327 self.draw()
315 328
316 329 update_figfile = False
317 330
318 331 if dataOut.ltctime >= self.tmax:
319 332 self.counter_imagwr = wr_period
320 333 self.isConfig = False
321 334 update_figfile = True
322 335
323 336 self.save(figpath=figpath,
324 337 figfile=figfile,
325 338 save=save,
326 339 ftp=ftp,
327 340 wr_period=wr_period,
328 341 thisDatetime=thisDatetime,
329 update_figfile=update_figfile) No newline at end of file
342 update_figfile=update_figfile)
343
344
345 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,848 +1,849
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import os, sys
8 8 import time, datetime
9 9 import numpy
10 10 import fnmatch
11 11 import glob
12 12 from time import sleep
13 13
14 14 try:
15 15 import pyfits
16 16 except ImportError as e:
17 17 print("Fits data cannot be used. Install pyfits module")
18 18
19 19 from xml.etree.ElementTree import ElementTree
20 20
21 21 from .jroIO_base import isRadarFolder, isNumber
22 22 from schainpy.model.data.jrodata import Fits
23 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
23 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecorator
24 from schainpy.utils import log
25
24 26
25 27 class PyFits(object):
26 28 name=None
27 29 format=None
28 30 array =None
29 31 data =None
30 32 thdulist=None
31 33 prihdr=None
32 34 hdu=None
33 35
34 36 def __init__(self):
35 37
36 38 pass
37 39
38 40 def setColF(self,name,format,array):
39 41 self.name=name
40 42 self.format=format
41 43 self.array=array
42 44 a1=numpy.array([self.array],dtype=numpy.float32)
43 45 self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
44 46 return self.col1
45 47
46 48 # def setColP(self,name,format,data):
47 49 # self.name=name
48 50 # self.format=format
49 51 # self.data=data
50 52 # a2=numpy.array([self.data],dtype=numpy.float32)
51 53 # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
52 54 # return self.col2
53 55
54 56
55 57 def writeData(self,name,format,data):
56 58 self.name=name
57 59 self.format=format
58 60 self.data=data
59 61 a2=numpy.array([self.data],dtype=numpy.float32)
60 62 self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
61 63 return self.col2
62 64
63 65 def cFImage(self,idblock,year,month,day,hour,minute,second):
64 66 self.hdu= pyfits.PrimaryHDU(idblock)
65 67 self.hdu.header.set("Year",year)
66 68 self.hdu.header.set("Month",month)
67 69 self.hdu.header.set("Day",day)
68 70 self.hdu.header.set("Hour",hour)
69 71 self.hdu.header.set("Minute",minute)
70 72 self.hdu.header.set("Second",second)
71 73 return self.hdu
72 74
73 75
74 76 def Ctable(self,colList):
75 77 self.cols=pyfits.ColDefs(colList)
76 78 self.tbhdu = pyfits.new_table(self.cols)
77 79 return self.tbhdu
78 80
79 81
80 82 def CFile(self,hdu,tbhdu):
81 83 self.thdulist=pyfits.HDUList([hdu,tbhdu])
82 84
83 85 def wFile(self,filename):
84 86 if os.path.isfile(filename):
85 87 os.remove(filename)
86 88 self.thdulist.writeto(filename)
87 89
88 90
89 91 class ParameterConf:
90 92 ELEMENTNAME = 'Parameter'
91 93 def __init__(self):
92 94 self.name = ''
93 95 self.value = ''
94 96
95 97 def readXml(self, parmElement):
96 98 self.name = parmElement.get('name')
97 99 self.value = parmElement.get('value')
98 100
99 101 def getElementName(self):
100 102 return self.ELEMENTNAME
101 103
102 104 class Metadata(object):
103 105
104 106 def __init__(self, filename):
105 107 self.parmConfObjList = []
106 108 self.readXml(filename)
107 109
108 110 def readXml(self, filename):
109 111 self.projectElement = None
110 112 self.procUnitConfObjDict = {}
111 113 self.projectElement = ElementTree().parse(filename)
112 114 self.project = self.projectElement.tag
113 115
114 116 parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
115 117
116 118 for parmElement in parmElementList:
117 119 parmConfObj = ParameterConf()
118 120 parmConfObj.readXml(parmElement)
119 121 self.parmConfObjList.append(parmConfObj)
120 122
121 123 class FitsWriter(Operation):
122 124 def __init__(self, **kwargs):
123 125 Operation.__init__(self, **kwargs)
124 126 self.isConfig = False
125 127 self.dataBlocksPerFile = None
126 128 self.blockIndex = 0
127 129 self.flagIsNewFile = 1
128 130 self.fitsObj = None
129 131 self.optchar = 'P'
130 132 self.ext = '.fits'
131 133 self.setFile = 0
132 134
133 135 def setFitsHeader(self, dataOut, metadatafile=None):
134 136
135 137 header_data = pyfits.PrimaryHDU()
136 138
137 139 header_data.header['EXPNAME'] = "RADAR DATA"
138 140 header_data.header['DATATYPE'] = "SPECTRA"
139 141 header_data.header['COMMENT'] = ""
140 142
141 143 if metadatafile:
142 144
143 145 metadata4fits = Metadata(metadatafile)
144 146
145 147 for parameter in metadata4fits.parmConfObjList:
146 148 parm_name = parameter.name
147 149 parm_value = parameter.value
148 150
149 151 header_data.header[parm_name] = parm_value
150 152
151 153 header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
152 154 header_data.header['CHANNELLIST'] = str(dataOut.channelList)
153 155 header_data.header['NCHANNELS'] = dataOut.nChannels
154 156 #header_data.header['HEIGHTS'] = dataOut.heightList
155 157 header_data.header['NHEIGHTS'] = dataOut.nHeights
156 158
157 159 header_data.header['IPPSECONDS'] = dataOut.ippSeconds
158 160 header_data.header['NCOHINT'] = dataOut.nCohInt
159 161 header_data.header['NINCOHINT'] = dataOut.nIncohInt
160 162 header_data.header['TIMEZONE'] = dataOut.timeZone
161 163 header_data.header['NBLOCK'] = self.blockIndex
162 164
163 165 header_data.writeto(self.filename)
164 166
165 167 self.addExtension(dataOut.heightList,'HEIGHTLIST')
166 168
167 169
168 170 def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None):
169 171
170 172 self.path = path
171 173 self.dataOut = dataOut
172 174 self.metadatafile = metadatafile
173 175 self.dataBlocksPerFile = dataBlocksPerFile
174 176
175 177 def open(self):
176 178 self.fitsObj = pyfits.open(self.filename, mode='update')
177 179
178 180
179 181 def addExtension(self, data, tagname):
180 182 self.open()
181 183 extension = pyfits.ImageHDU(data=data, name=tagname)
182 184 #extension.header['TAG'] = tagname
183 185 self.fitsObj.append(extension)
184 186 self.write()
185 187
186 188 def addData(self, data):
187 189 self.open()
188 190 extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
189 191 extension.header['UTCTIME'] = self.dataOut.utctime
190 192 self.fitsObj.append(extension)
191 193 self.blockIndex += 1
192 194 self.fitsObj[0].header['NBLOCK'] = self.blockIndex
193 195
194 196 self.write()
195 197
196 198 def write(self):
197 199
198 200 self.fitsObj.flush(verbose=True)
199 201 self.fitsObj.close()
200 202
201 203
202 204 def setNextFile(self):
203 205
204 206 ext = self.ext
205 207 path = self.path
206 208
207 209 timeTuple = time.localtime( self.dataOut.utctime)
208 210 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
209 211
210 212 fullpath = os.path.join( path, subfolder )
211 213 if not( os.path.exists(fullpath) ):
212 214 os.mkdir(fullpath)
213 215 self.setFile = -1 #inicializo mi contador de seteo
214 216 else:
215 217 filesList = os.listdir( fullpath )
216 218 if len( filesList ) > 0:
217 219 filesList = sorted( filesList, key=str.lower )
218 220 filen = filesList[-1]
219 221
220 222 if isNumber( filen[8:11] ):
221 223 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
222 224 else:
223 225 self.setFile = -1
224 226 else:
225 227 self.setFile = -1 #inicializo mi contador de seteo
226 228
227 229 setFile = self.setFile
228 230 setFile += 1
229 231
230 232 thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
231 233 timeTuple.tm_year,
232 234 timeTuple.tm_yday,
233 235 setFile,
234 236 ext )
235 237
236 238 filename = os.path.join( path, subfolder, thisFile )
237 239
238 240 self.blockIndex = 0
239 241 self.filename = filename
240 242 self.setFile = setFile
241 243 self.flagIsNewFile = 1
242 244
243 245 print('Writing the file: %s'%self.filename)
244 246
245 247 self.setFitsHeader(self.dataOut, self.metadatafile)
246 248
247 249 return 1
248 250
249 251 def writeBlock(self):
250 252 self.addData(self.dataOut.data_spc)
251 253 self.flagIsNewFile = 0
252 254
253 255
254 256 def __setNewBlock(self):
255 257
256 258 if self.flagIsNewFile:
257 259 return 1
258 260
259 261 if self.blockIndex < self.dataBlocksPerFile:
260 262 return 1
261 263
262 264 if not( self.setNextFile() ):
263 265 return 0
264 266
265 267 return 1
266 268
267 269 def writeNextBlock(self):
268 270 if not( self.__setNewBlock() ):
269 271 return 0
270 272 self.writeBlock()
271 273 return 1
272 274
273 275 def putData(self):
274 276 if self.flagIsNewFile:
275 277 self.setNextFile()
276 278 self.writeNextBlock()
277 279
278 280 def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs):
279 281 if not(self.isConfig):
280 282 self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs)
281 283 self.isConfig = True
282 284 self.putData()
283 285
284
286 @MPDecorator
285 287 class FitsReader(ProcessingUnit):
286 288
287 289 # __TIMEZONE = time.timezone
288 290
289 291 expName = None
290 292 datetimestr = None
291 293 utc = None
292 294 nChannels = None
293 295 nSamples = None
294 296 dataBlocksPerFile = None
295 297 comments = None
296 298 lastUTTime = None
297 299 header_dict = None
298 300 data = None
299 301 data_header_dict = None
300 302
301 def __init__(self, **kwargs):
302 ProcessingUnit.__init__(self, **kwargs)
303 def __init__(self):#, **kwargs):
304 ProcessingUnit.__init__(self)#, **kwargs)
303 305 self.isConfig = False
304 306 self.ext = '.fits'
305 307 self.setFile = 0
306 308 self.flagNoMoreFiles = 0
307 309 self.flagIsNewFile = 1
308 310 self.flagDiscontinuousBlock = None
309 311 self.fileIndex = None
310 312 self.filename = None
311 313 self.fileSize = None
312 314 self.fitsObj = None
313 315 self.timeZone = None
314 316 self.nReadBlocks = 0
315 317 self.nTotalBlocks = 0
316 318 self.dataOut = self.createObjByDefault()
317 319 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
318 320 self.blockIndex = 1
319 321
320 322 def createObjByDefault(self):
321 323
322 324 dataObj = Fits()
323 325
324 326 return dataObj
325 327
326 328 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
327 329 try:
328 330 fitsObj = pyfits.open(filename,'readonly')
329 331 except:
330 332 print("File %s can't be opened" %(filename))
331 333 return None
332 334
333 335 header = fitsObj[0].header
334 336 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
335 337 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
336 338
337 339 ltc = utc
338 340 if useLocalTime:
339 341 ltc -= time.timezone
340 342 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
341 343 thisTime = thisDatetime.time()
342 344
343 345 if not ((startTime <= thisTime) and (endTime > thisTime)):
344 346 return None
345 347
346 348 return thisDatetime
347 349
348 350 def __setNextFileOnline(self):
349 351 raise NotImplementedError
350 352
351 353 def __setNextFileOffline(self):
352 354 idFile = self.fileIndex
353 355
354 356 while (True):
355 357 idFile += 1
356 358 if not(idFile < len(self.filenameList)):
357 359 self.flagNoMoreFiles = 1
358 360 print("No more Files")
359 361 return 0
360 362
361 363 filename = self.filenameList[idFile]
362 364
363 365 # if not(self.__verifyFile(filename)):
364 366 # continue
365 367
366 368 fileSize = os.path.getsize(filename)
367 369 fitsObj = pyfits.open(filename,'readonly')
368 370 break
369 371
370 372 self.flagIsNewFile = 1
371 373 self.fileIndex = idFile
372 374 self.filename = filename
373 375 self.fileSize = fileSize
374 376 self.fitsObj = fitsObj
375 377 self.blockIndex = 0
376 378 print("Setting the file: %s"%self.filename)
377 379
378 380 return 1
379 381
380 382 def __setValuesFromHeader(self):
381 383
382 384 self.dataOut.header = self.header_dict
383 385 self.dataOut.expName = self.expName
384 386
385 387 self.dataOut.timeZone = self.timeZone
386 388 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
387 389 self.dataOut.comments = self.comments
388 390 # self.dataOut.timeInterval = self.timeInterval
389 391 self.dataOut.channelList = self.channelList
390 392 self.dataOut.heightList = self.heightList
391 393
392 394 self.dataOut.nCohInt = self.nCohInt
393 395 self.dataOut.nIncohInt = self.nIncohInt
394
395 self.dataOut.ippSeconds = self.ippSeconds
396 self.dataOut.ipp_sec = self.ippSeconds
396 397
397 398 def readHeader(self):
398 399 headerObj = self.fitsObj[0]
399 400
400 401 self.header_dict = headerObj.header
401 402 if 'EXPNAME' in list(headerObj.header.keys()):
402 403 self.expName = headerObj.header['EXPNAME']
403 404
404 405 if 'DATATYPE' in list(headerObj.header.keys()):
405 406 self.dataType = headerObj.header['DATATYPE']
406 407
407 408 self.datetimestr = headerObj.header['DATETIME']
408 409 channelList = headerObj.header['CHANNELLIST']
409 410 channelList = channelList.split('[')
410 411 channelList = channelList[1].split(']')
411 412 channelList = channelList[0].split(',')
412 413 channelList = [int(ch) for ch in channelList]
413 414 self.channelList = channelList
414 415 self.nChannels = headerObj.header['NCHANNELS']
415 416 self.nHeights = headerObj.header['NHEIGHTS']
416 417 self.ippSeconds = headerObj.header['IPPSECONDS']
417 418 self.nCohInt = headerObj.header['NCOHINT']
418 419 self.nIncohInt = headerObj.header['NINCOHINT']
419 420 self.dataBlocksPerFile = headerObj.header['NBLOCK']
420 421 self.timeZone = headerObj.header['TIMEZONE']
421 422
422 423 # self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
423 424
424 425 if 'COMMENT' in list(headerObj.header.keys()):
425 426 self.comments = headerObj.header['COMMENT']
426 427
427 428 self.readHeightList()
428 429
429 430 def readHeightList(self):
430 431 self.blockIndex = self.blockIndex + 1
431 432 obj = self.fitsObj[self.blockIndex]
432 433 self.heightList = obj.data
433 434 self.blockIndex = self.blockIndex + 1
434 435
435 436 def readExtension(self):
436 437 obj = self.fitsObj[self.blockIndex]
437 438 self.heightList = obj.data
438 439 self.blockIndex = self.blockIndex + 1
439 440
440 441 def setNextFile(self):
441 442
442 443 if self.online:
443 444 newFile = self.__setNextFileOnline()
444 445 else:
445 446 newFile = self.__setNextFileOffline()
446 447
447 448 if not(newFile):
448 449 return 0
449 450
450 451 self.readHeader()
451 452 self.__setValuesFromHeader()
452 453 self.nReadBlocks = 0
453 454 # self.blockIndex = 1
454 455 return 1
455 456
456 457 def searchFilesOffLine(self,
457 458 path,
458 459 startDate,
459 460 endDate,
460 461 startTime=datetime.time(0,0,0),
461 462 endTime=datetime.time(23,59,59),
462 463 set=None,
463 464 expLabel='',
464 465 ext='.fits',
465 466 walk=True):
466 467
467 468 pathList = []
468 469
469 470 if not walk:
470 471 pathList.append(path)
471 472
472 473 else:
473 474 dirList = []
474 475 for thisPath in os.listdir(path):
475 476 if not os.path.isdir(os.path.join(path,thisPath)):
476 477 continue
477 478 if not isRadarFolder(thisPath):
478 479 continue
479 480
480 481 dirList.append(thisPath)
481 482
482 483 if not(dirList):
483 484 return None, None
484 485
485 486 thisDate = startDate
486 487
487 488 while(thisDate <= endDate):
488 489 year = thisDate.timetuple().tm_year
489 490 doy = thisDate.timetuple().tm_yday
490 491
491 492 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
492 493 if len(matchlist) == 0:
493 494 thisDate += datetime.timedelta(1)
494 495 continue
495 496 for match in matchlist:
496 497 pathList.append(os.path.join(path,match,expLabel))
497 498
498 499 thisDate += datetime.timedelta(1)
499 500
500 501 if pathList == []:
501 502 print("Any folder was found for the date range: %s-%s" %(startDate, endDate))
502 503 return None, None
503 504
504 505 print("%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate))
505 506
506 507 filenameList = []
507 508 datetimeList = []
508 509
509 510 for i in range(len(pathList)):
510 511
511 512 thisPath = pathList[i]
512 513
513 514 fileList = glob.glob1(thisPath, "*%s" %ext)
514 515 fileList.sort()
515 516
516 517 for thisFile in fileList:
517 518
518 519 filename = os.path.join(thisPath,thisFile)
519 520 thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
520 521
521 522 if not(thisDatetime):
522 523 continue
523 524
524 525 filenameList.append(filename)
525 526 datetimeList.append(thisDatetime)
526 527
527 528 if not(filenameList):
528 529 print("Any file was found for the time range %s - %s" %(startTime, endTime))
529 530 return None, None
530 531
531 532 print("%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime))
532 533 print()
533 534
534 535 for i in range(len(filenameList)):
535 536 print("%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()))
536 537
537 538 self.filenameList = filenameList
538 539 self.datetimeList = datetimeList
539 540
540 541 return pathList, filenameList
541 542
542 543 def setup(self, path=None,
543 544 startDate=None,
544 545 endDate=None,
545 546 startTime=datetime.time(0,0,0),
546 547 endTime=datetime.time(23,59,59),
547 548 set=0,
548 549 expLabel = "",
549 550 ext = None,
550 551 online = False,
551 552 delay = 60,
552 553 walk = True):
553 554
554 555 if path == None:
555 556 raise ValueError("The path is not valid")
556 557
557 558 if ext == None:
558 559 ext = self.ext
559 560
560 561 if not(online):
561 562 print("Searching files in offline mode ...")
562 563 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
563 564 startTime=startTime, endTime=endTime,
564 565 set=set, expLabel=expLabel, ext=ext,
565 566 walk=walk)
566 567
567 568 if not(pathList):
568 569 print("No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
569 570 datetime.datetime.combine(startDate,startTime).ctime(),
570 571 datetime.datetime.combine(endDate,endTime).ctime()))
571 572
572 573 sys.exit(-1)
573 574
574 575 self.fileIndex = -1
575 576 self.pathList = pathList
576 577 self.filenameList = filenameList
577 578
578 579 self.online = online
579 580 self.delay = delay
580 581 ext = ext.lower()
581 582 self.ext = ext
582 583
583 584 if not(self.setNextFile()):
584 585 if (startDate!=None) and (endDate!=None):
585 586 print("No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
586 587 elif startDate != None:
587 588 print("No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()))
588 589 else:
589 590 print("No files")
590 591
591 592 sys.exit(-1)
592 593
593 594
594 595
595 596 def readBlock(self):
596 597 dataObj = self.fitsObj[self.blockIndex]
597 598
598 599 self.data = dataObj.data
599 600 self.data_header_dict = dataObj.header
600 601 self.utc = self.data_header_dict['UTCTIME']
601 602
602 603 self.flagIsNewFile = 0
603 604 self.blockIndex += 1
604 605 self.nTotalBlocks += 1
605 606 self.nReadBlocks += 1
606 607
607 608 return 1
608 609
609 610 def __jumpToLastBlock(self):
610 611 raise NotImplementedError
611 612
612 613 def __waitNewBlock(self):
613 614 """
614 615 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
615 616
616 617 Si el modo de lectura es OffLine siempre retorn 0
617 618 """
618 619 if not self.online:
619 620 return 0
620 621
621 622 if (self.nReadBlocks >= self.dataBlocksPerFile):
622 623 return 0
623 624
624 625 currentPointer = self.fp.tell()
625 626
626 627 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
627 628
628 629 for nTries in range( self.nTries ):
629 630
630 631 self.fp.close()
631 632 self.fp = open( self.filename, 'rb' )
632 633 self.fp.seek( currentPointer )
633 634
634 635 self.fileSize = os.path.getsize( self.filename )
635 636 currentSize = self.fileSize - currentPointer
636 637
637 638 if ( currentSize >= neededSize ):
638 639 self.__rdBasicHeader()
639 640 return 1
640 641
641 642 print("\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1))
642 643 sleep( self.delay )
643 644
644 645
645 646 return 0
646 647
647 648 def __setNewBlock(self):
648 649
649 650 if self.online:
650 651 self.__jumpToLastBlock()
651 652
652 653 if self.flagIsNewFile:
653 654 return 1
654 655
655 656 self.lastUTTime = self.utc
656 657
657 658 if self.online:
658 659 if self.__waitNewBlock():
659 660 return 1
660 661
661 662 if self.nReadBlocks < self.dataBlocksPerFile:
662 663 return 1
663 664
664 665 if not(self.setNextFile()):
665 666 return 0
666 667
667 668 deltaTime = self.utc - self.lastUTTime
668 669
669 670 self.flagDiscontinuousBlock = 0
670 671
671 672 if deltaTime > self.maxTimeStep:
672 673 self.flagDiscontinuousBlock = 1
673 674
674 675 return 1
675 676
676 677
677 678 def readNextBlock(self):
678 679 if not(self.__setNewBlock()):
679 680 return 0
680 681
681 682 if not(self.readBlock()):
682 683 return 0
683 684
684 685 return 1
685 686
686 687 def printInfo(self):
687 688
688 689 pass
689 690
690 691 def getData(self):
691 692
692 693 if self.flagNoMoreFiles:
693 694 self.dataOut.flagNoData = True
694 print('Process finished')
695 return 0
695 return (0, 'No more files')
696 696
697 697 self.flagDiscontinuousBlock = 0
698 698 self.flagIsNewBlock = 0
699 699
700 700 if not(self.readNextBlock()):
701 return 0
701 return (1, 'Error reading data')
702 702
703 703 if self.data is None:
704 704 self.dataOut.flagNoData = True
705 return 0
705 return (0, 'No more data')
706 706
707 707 self.dataOut.data = self.data
708 708 self.dataOut.data_header = self.data_header_dict
709 709 self.dataOut.utctime = self.utc
710 710
711 711 # self.dataOut.header = self.header_dict
712 712 # self.dataOut.expName = self.expName
713 713 # self.dataOut.nChannels = self.nChannels
714 714 # self.dataOut.timeZone = self.timeZone
715 715 # self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
716 716 # self.dataOut.comments = self.comments
717 717 # # self.dataOut.timeInterval = self.timeInterval
718 718 # self.dataOut.channelList = self.channelList
719 719 # self.dataOut.heightList = self.heightList
720 720 self.dataOut.flagNoData = False
721
722 return self.dataOut.data
721 # return self.dataOut.data
723 722
724 723 def run(self, **kwargs):
725 724
726 725 if not(self.isConfig):
727 726 self.setup(**kwargs)
728 727 self.isConfig = True
729 728
730 729 self.getData()
731 730
731 @MPDecorator
732 732 class SpectraHeisWriter(Operation):
733 733 # set = None
734 734 setFile = None
735 735 idblock = None
736 736 doypath = None
737 737 subfolder = None
738 738
739 def __init__(self, **kwargs):
740 Operation.__init__(self, **kwargs)
739 def __init__(self):#, **kwargs):
740 Operation.__init__(self)#, **kwargs)
741 741 self.wrObj = PyFits()
742 742 # self.dataOut = dataOut
743 743 self.nTotalBlocks=0
744 744 # self.set = None
745 745 self.setFile = None
746 746 self.idblock = 0
747 747 self.wrpath = None
748 748 self.doypath = None
749 749 self.subfolder = None
750 750 self.isConfig = False
751 751
752 752 def isNumber(str):
753 753 """
754 754 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
755 755
756 756 Excepciones:
757 757 Si un determinado string no puede ser convertido a numero
758 758 Input:
759 759 str, string al cual se le analiza para determinar si convertible a un numero o no
760 760
761 761 Return:
762 762 True : si el string es uno numerico
763 763 False : no es un string numerico
764 764 """
765 765 try:
766 766 float( str )
767 767 return True
768 768 except:
769 769 return False
770 770
771 771 def setup(self, dataOut, wrpath):
772 772
773 773 if not(os.path.exists(wrpath)):
774 774 os.mkdir(wrpath)
775 775
776 776 self.wrpath = wrpath
777 777 # self.setFile = 0
778 778 self.dataOut = dataOut
779 779
780 780 def putData(self):
781 781 name= time.localtime( self.dataOut.utctime)
782 782 ext=".fits"
783 783
784 784 if self.doypath == None:
785 785 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
786 786 self.doypath = os.path.join( self.wrpath, self.subfolder )
787 787 os.mkdir(self.doypath)
788 788
789 789 if self.setFile == None:
790 790 # self.set = self.dataOut.set
791 791 self.setFile = 0
792 792 # if self.set != self.dataOut.set:
793 793 ## self.set = self.dataOut.set
794 794 # self.setFile = 0
795 795
796 796 #make the filename
797 797 thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
798 798
799 799 filename = os.path.join(self.wrpath,self.subfolder, thisFile)
800 800
801 801 idblock = numpy.array([self.idblock],dtype="int64")
802 802 header=self.wrObj.cFImage(idblock=idblock,
803 803 year=time.gmtime(self.dataOut.utctime).tm_year,
804 804 month=time.gmtime(self.dataOut.utctime).tm_mon,
805 805 day=time.gmtime(self.dataOut.utctime).tm_mday,
806 806 hour=time.gmtime(self.dataOut.utctime).tm_hour,
807 807 minute=time.gmtime(self.dataOut.utctime).tm_min,
808 808 second=time.gmtime(self.dataOut.utctime).tm_sec)
809 809
810 810 c=3E8
811 811 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
812 812 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
813 813
814 814 colList = []
815 815
816 816 colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
817 817
818 818 colList.append(colFreq)
819 819
820 820 nchannel=self.dataOut.nChannels
821 821
822 822 for i in range(nchannel):
823 823 col = self.wrObj.writeData(name="PCh"+str(i+1),
824 824 format=str(self.dataOut.nFFTPoints)+'E',
825 825 data=10*numpy.log10(self.dataOut.data_spc[i,:]))
826 826
827 827 colList.append(col)
828 828
829 829 data=self.wrObj.Ctable(colList=colList)
830 830
831 831 self.wrObj.CFile(header,data)
832 832
833 833 self.wrObj.wFile(filename)
834 834
835 835 #update the setFile
836 836 self.setFile += 1
837 837 self.idblock += 1
838 838
839 839 return 1
840 840
841 841 def run(self, dataOut, **kwargs):
842 842
843 843 if not(self.isConfig):
844 844
845 845 self.setup(dataOut, **kwargs)
846 846 self.isConfig = True
847 847
848 self.putData() No newline at end of file
848 self.putData()
849 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,376 +1,380
1 1 '''
2 2 Updated for multiprocessing
3 3 Author : Sergio Cortez
4 4 Jan 2018
5 5 Abstract:
6 6 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
7 7 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
8 8 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
9 9
10 10 Based on:
11 11 $Author: murco $
12 12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
13 13 '''
14 14
15 15 import inspect
16 16 import zmq
17 17 import time
18 18 import pickle
19 19 import os
20 20 from multiprocessing import Process
21 21 from zmq.utils.monitor import recv_monitor_message
22 22
23 23 from schainpy.utils import log
24 24
25 25
26 26 class ProcessingUnit(object):
27 27
28 28 """
29 29 Update - Jan 2018 - MULTIPROCESSING
30 30 All the "call" methods present in the previous base were removed.
31 31 The majority of operations are independant processes, thus
32 32 the decorator is in charge of communicate the operation processes
33 33 with the proccessing unit via IPC.
34 34
35 35 The constructor does not receive any argument. The remaining methods
36 36 are related with the operations to execute.
37 37
38 38
39 39 """
40 40
41 41 def __init__(self):
42 42
43 43 self.dataIn = None
44 44 self.dataOut = None
45 45 self.isConfig = False
46 46 self.operations = []
47 47 self.plots = []
48 48
49 49 def getAllowedArgs(self):
50 50 if hasattr(self, '__attrs__'):
51 51 return self.__attrs__
52 52 else:
53 53 return inspect.getargspec(self.run).args
54 54
55 55 def addOperation(self, conf, operation):
56 56 """
57 57 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
58 58 posses the id of the operation process (IPC purposes)
59 59
60 60 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
61 61 identificador asociado a este objeto.
62 62
63 63 Input:
64 64
65 65 object : objeto de la clase "Operation"
66 66
67 67 Return:
68 68
69 69 objId : identificador del objeto, necesario para comunicar con master(procUnit)
70 70 """
71 71
72 72 self.operations.append(
73 73 (operation, conf.type, conf.id, conf.getKwargs()))
74 74
75 75 if 'plot' in self.name.lower():
76 76 self.plots.append(operation.CODE)
77 77
78 78 def getOperationObj(self, objId):
79 79
80 80 if objId not in list(self.operations.keys()):
81 81 return None
82 82
83 83 return self.operations[objId]
84 84
85 85 def operation(self, **kwargs):
86 86 """
87 87 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
88 88 atributos del objeto dataOut
89 89
90 90 Input:
91 91
92 92 **kwargs : Diccionario de argumentos de la funcion a ejecutar
93 93 """
94 94
95 95 raise NotImplementedError
96 96
97 97 def setup(self):
98 98
99 99 raise NotImplementedError
100 100
101 101 def run(self):
102 102
103 103 raise NotImplementedError
104 104
105 105 def close(self):
106 106
107 107 return
108 108
109 109
110 110 class Operation(object):
111 111
112 112 """
113 113 Update - Jan 2018 - MULTIPROCESSING
114 114
115 115 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
116 116 The constructor doe snot receive any argument, neither the baseclass.
117 117
118 118
119 119 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
120 120 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
121 121 acumulacion dentro de esta clase
122 122
123 123 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
124 124
125 125 """
126 126
127 127 def __init__(self):
128 128
129 129 self.id = None
130 130 self.isConfig = False
131 131
132 132 if not hasattr(self, 'name'):
133 133 self.name = self.__class__.__name__
134 134
135 135 def getAllowedArgs(self):
136 136 if hasattr(self, '__attrs__'):
137 137 return self.__attrs__
138 138 else:
139 139 return inspect.getargspec(self.run).args
140 140
141 141 def setup(self):
142 142
143 143 self.isConfig = True
144 144
145 145 raise NotImplementedError
146 146
147 147 def run(self, dataIn, **kwargs):
148 148 """
149 149 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
150 150 atributos del objeto dataIn.
151 151
152 152 Input:
153 153
154 154 dataIn : objeto del tipo JROData
155 155
156 156 Return:
157 157
158 158 None
159 159
160 160 Affected:
161 161 __buffer : buffer de recepcion de datos.
162 162
163 163 """
164 164 if not self.isConfig:
165 165 self.setup(**kwargs)
166 166
167 167 raise NotImplementedError
168 168
169 169 def close(self):
170 170
171 171 return
172 172
173 173
174 174 def MPDecorator(BaseClass):
175 175 """
176 176 Multiprocessing class decorator
177 177
178 178 This function add multiprocessing features to a BaseClass. Also, it handle
179 179 the communication beetween processes (readers, procUnits and operations).
180 180 """
181 181
182 182 class MPClass(BaseClass, Process):
183 183
184 184 def __init__(self, *args, **kwargs):
185 185 super(MPClass, self).__init__()
186 186 Process.__init__(self)
187 187 self.operationKwargs = {}
188 188 self.args = args
189 189 self.kwargs = kwargs
190 190 self.sender = None
191 191 self.receiver = None
192 192 self.name = BaseClass.__name__
193 193 self.start_time = time.time()
194 194
195 195 if len(self.args) is 3:
196 196 self.typeProc = "ProcUnit"
197 197 self.id = args[0]
198 198 self.inputId = args[1]
199 199 self.project_id = args[2]
200 200 elif len(self.args) is 2:
201 201 self.id = args[0]
202 202 self.inputId = args[0]
203 203 self.project_id = args[1]
204 204 self.typeProc = "Operation"
205 205
206 206 def subscribe(self):
207 207 '''
208 208 This function create a socket to receive objects from the
209 209 topic `inputId`.
210 210 '''
211 211
212 212 c = zmq.Context()
213 213 self.receiver = c.socket(zmq.SUB)
214 214 self.receiver.connect(
215 215 'ipc:///tmp/schain/{}_pub'.format(self.project_id))
216 216 self.receiver.setsockopt(zmq.SUBSCRIBE, self.inputId.encode())
217
217
218 218 def listen(self):
219 219 '''
220 220 This function waits for objects and deserialize using pickle
221 221 '''
222
222
223 223 data = pickle.loads(self.receiver.recv_multipart()[1])
224
224
225 225 return data
226 226
227 227 def set_publisher(self):
228 228 '''
229 229 This function create a socket for publishing purposes.
230 230 '''
231 231
232 232 time.sleep(1)
233 233 c = zmq.Context()
234 234 self.sender = c.socket(zmq.PUB)
235 235 self.sender.connect(
236 236 'ipc:///tmp/schain/{}_sub'.format(self.project_id))
237 237
238 238 def publish(self, data, id):
239 239 '''
240 240 This function publish an object, to a specific topic.
241 241 '''
242 242 self.sender.send_multipart([str(id).encode(), pickle.dumps(data)])
243 243
244 244 def runReader(self):
245 245 '''
246 246 Run fuction for read units
247 247 '''
248 248 while True:
249 249
250 250 BaseClass.run(self, **self.kwargs)
251 251
252 252 for op, optype, opId, kwargs in self.operations:
253 253 if optype == 'self':
254 254 op(**kwargs)
255 255 elif optype == 'other':
256 256 self.dataOut = op.run(self.dataOut, **self.kwargs)
257 257 elif optype == 'external':
258 258 self.publish(self.dataOut, opId)
259 259
260 260 if self.dataOut.flagNoData and self.dataOut.error is None:
261 261 continue
262 262
263 263 self.publish(self.dataOut, self.id)
264 264
265 265 if self.dataOut.error:
266 266 if self.dataOut.error[0] == -1:
267 267 log.error(self.dataOut.error[1], self.name)
268 268 if self.dataOut.error[0] == 1:
269 269 log.success(self.dataOut.error[1], self.name)
270 270 # self.sender.send_multipart([str(self.project_id).encode(), 'end'.encode()])
271 271 break
272 272
273 273 time.sleep(1)
274 274
275 275 def runProc(self):
276 276 '''
277 277 Run function for proccessing units
278 278 '''
279 279
280 280 while True:
281 self.dataIn = self.listen()
281 self.dataIn = self.listen()
282 282
283 283 if self.dataIn.flagNoData and self.dataIn.error is None:
284 284 continue
285
285
286 286 BaseClass.run(self, **self.kwargs)
287 287
288 if self.dataOut.flagNoData:
289 continue
290
288 291 for op, optype, opId, kwargs in self.operations:
289 292 if optype == 'self':
290 293 op(**kwargs)
291 294 elif optype == 'other':
292 295 self.dataOut = op.run(self.dataOut, **kwargs)
293 296 elif optype == 'external':
294 297 self.publish(self.dataOut, opId)
295 298
296 299 self.publish(self.dataOut, self.id)
297 300 if self.dataIn.error:
298 301 break
299 302
300 303 time.sleep(1)
301 304
302 305 def runOp(self):
303 306 '''
304 307 Run function for external operations (this operations just receive data
305 308 ex: plots, writers, publishers)
306 309 '''
307
310
308 311 while True:
309 312
310 313 dataOut = self.listen()
311 314
312 315 BaseClass.run(self, dataOut, **self.kwargs)
313 316
314 317 if dataOut.error:
315 318 break
316 319 time.sleep(1)
317 320
318 321 def run(self):
319
320 322 if self.typeProc is "ProcUnit":
321 323
322 324 if self.inputId is not None:
325
323 326 self.subscribe()
327
324 328 self.set_publisher()
325 329
326 330 if 'Reader' not in BaseClass.__name__:
327 331 self.runProc()
328 332 else:
329 333 self.runReader()
330 334
331 335 elif self.typeProc is "Operation":
332 336
333 337 self.subscribe()
334 338 self.runOp()
335 339
336 340 else:
337 341 raise ValueError("Unknown type")
338 342
339 343 self.close()
340 344
341 345 def event_monitor(self, monitor):
342 346
343 347 events = {}
344 348
345 349 for name in dir(zmq):
346 350 if name.startswith('EVENT_'):
347 351 value = getattr(zmq, name)
348 352 events[value] = name
349 353
350 354 while monitor.poll():
351 355 evt = recv_monitor_message(monitor)
352 356 if evt['event'] == 32:
353 357 self.connections += 1
354 358 if evt['event'] == 512:
355 359 pass
356 360
357 361 evt.update({'description': events[evt['event']]})
358 362
359 363 if evt['event'] == zmq.EVENT_MONITOR_STOPPED:
360 364 break
361 365 monitor.close()
362 366 print('event monitor thread done!')
363 367
364 368 def close(self):
365 369
366 370 BaseClass.close(self)
367 371
368 372 if self.sender:
369 373 self.sender.close()
370 374
371 375 if self.receiver:
372 376 self.receiver.close()
373 377
374 378 log.success('Done...(Time:{:4.2f} secs)'.format(time.time()-self.start_time), self.name)
375 379
376 380 return MPClass
Show file before | Show file after | Hide comments
@@ -1,344 +1,350
1 1 import numpy
2 2
3 from .jroproc_base import ProcessingUnit, Operation
3 from .jroproc_base import ProcessingUnit, Operation, MPDecorator
4 4 from schainpy.model.data.jrodata import SpectraHeis
5 from schainpy.utils import log
5 6
7
8 @MPDecorator
6 9 class SpectraHeisProc(ProcessingUnit):
7 10
8 def __init__(self, **kwargs):
11 def __init__(self):#, **kwargs):
9 12
10 ProcessingUnit.__init__(self, **kwargs)
13 ProcessingUnit.__init__(self)#, **kwargs)
11 14
12 15 # self.buffer = None
13 16 # self.firstdatatime = None
14 17 # self.profIndex = 0
15 18 self.dataOut = SpectraHeis()
16 19
17 20 def __updateObjFromVoltage(self):
18 21
19 22 self.dataOut.timeZone = self.dataIn.timeZone
20 23 self.dataOut.dstFlag = self.dataIn.dstFlag
21 24 self.dataOut.errorCount = self.dataIn.errorCount
22 25 self.dataOut.useLocalTime = self.dataIn.useLocalTime
23 26
24 27 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()#
25 28 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()#
26 29 self.dataOut.channelList = self.dataIn.channelList
27 30 self.dataOut.heightList = self.dataIn.heightList
28 31 # self.dataOut.dtype = self.dataIn.dtype
29 32 self.dataOut.dtype = numpy.dtype([('real','<f4'),('imag','<f4')])
30 33 # self.dataOut.nHeights = self.dataIn.nHeights
31 34 # self.dataOut.nChannels = self.dataIn.nChannels
32 35 self.dataOut.nBaud = self.dataIn.nBaud
33 36 self.dataOut.nCode = self.dataIn.nCode
34 37 self.dataOut.code = self.dataIn.code
35 38 # self.dataOut.nProfiles = 1
36 39 self.dataOut.ippFactor = 1
37 40 self.dataOut.noise_estimation = None
38 41 # self.dataOut.nProfiles = self.dataOut.nFFTPoints
39 42 self.dataOut.nFFTPoints = self.dataIn.nHeights
40 43 # self.dataOut.channelIndexList = self.dataIn.channelIndexList
41 44 # self.dataOut.flagNoData = self.dataIn.flagNoData
42 45 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
43 46 self.dataOut.utctime = self.dataIn.utctime
44 47 # self.dataOut.utctime = self.firstdatatime
45 48 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData #asumo q la data esta decodificada
46 49 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData #asumo q la data esta sin flip
47 50 # self.dataOut.flagShiftFFT = self.dataIn.flagShiftFFT
48 51 self.dataOut.nCohInt = self.dataIn.nCohInt
49 52 self.dataOut.nIncohInt = 1
50 53 # self.dataOut.ippSeconds= self.dataIn.ippSeconds
51 54 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
52 55
53 56 # self.dataOut.timeInterval = self.dataIn.timeInterval*self.dataOut.nIncohInt
54 57 # self.dataOut.set=self.dataIn.set
55 58 # self.dataOut.deltaHeight=self.dataIn.deltaHeight
56 59
57 60
58 61 def __updateObjFromFits(self):
59 62
60 63 self.dataOut.utctime = self.dataIn.utctime
61 64 # self.dataOut.channelIndexList = self.dataIn.channelIndexList
62 65
63 66 self.dataOut.channelList = self.dataIn.channelList
64 67 self.dataOut.heightList = self.dataIn.heightList
65 68 self.dataOut.data_spc = self.dataIn.data
66 69 self.dataOut.ippSeconds = self.dataIn.ippSeconds
67 70 self.dataOut.nCohInt = self.dataIn.nCohInt
68 71 self.dataOut.nIncohInt = self.dataIn.nIncohInt
69 72 # self.dataOut.timeInterval = self.dataIn.timeInterval
70 73 self.dataOut.timeZone = self.dataIn.timeZone
71 74 self.dataOut.useLocalTime = True
72 75 # self.dataOut.
73 76 # self.dataOut.
74 77
75 78 def __getFft(self):
76 79
77 80 fft_volt = numpy.fft.fft(self.dataIn.data, axis=1)
78 81 fft_volt = numpy.fft.fftshift(fft_volt,axes=(1,))
79 82 spc = numpy.abs(fft_volt * numpy.conjugate(fft_volt))/(self.dataOut.nFFTPoints)
80 83 self.dataOut.data_spc = spc
81 84
82 85 def run(self):
83 86
84 87 self.dataOut.flagNoData = True
85 88
86 89 if self.dataIn.type == "Fits":
87 90 self.__updateObjFromFits()
88 91 self.dataOut.flagNoData = False
89 return
92 return
90 93
91 94 if self.dataIn.type == "SpectraHeis":
92 95 self.dataOut.copy(self.dataIn)
93 96 return
94 97
95 98 if self.dataIn.type == "Voltage":
96 99 self.__updateObjFromVoltage()
97 100 self.__getFft()
98 101 self.dataOut.flagNoData = False
99 102
100 103 return
101 104
102 105 raise ValueError("The type object %s is not valid"%(self.dataIn.type))
103 106
104 107
105 108 def selectChannels(self, channelList):
106 109
107 110 channelIndexList = []
108 111
109 112 for channel in channelList:
110 113 index = self.dataOut.channelList.index(channel)
111 114 channelIndexList.append(index)
112 115
113 116 self.selectChannelsByIndex(channelIndexList)
114 117
115 118 def selectChannelsByIndex(self, channelIndexList):
116 119 """
117 120 Selecciona un bloque de datos en base a canales segun el channelIndexList
118 121
119 122 Input:
120 123 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
121 124
122 125 Affected:
123 126 self.dataOut.data
124 127 self.dataOut.channelIndexList
125 128 self.dataOut.nChannels
126 129 self.dataOut.m_ProcessingHeader.totalSpectra
127 130 self.dataOut.systemHeaderObj.numChannels
128 131 self.dataOut.m_ProcessingHeader.blockSize
129 132
130 133 Return:
131 134 None
132 135 """
133 136
134 137 for channelIndex in channelIndexList:
135 138 if channelIndex not in self.dataOut.channelIndexList:
136 139 print(channelIndexList)
137 140 raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
138 141
139 142 # nChannels = len(channelIndexList)
140 143
141 144 data_spc = self.dataOut.data_spc[channelIndexList,:]
142 145
143 146 self.dataOut.data_spc = data_spc
144 147 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
145 148
146 149 return 1
147 150
151
148 152 class IncohInt4SpectraHeis(Operation):
149 153
150 154 isConfig = False
151 155
152 156 __profIndex = 0
153 157 __withOverapping = False
154 158
155 159 __byTime = False
156 160 __initime = None
157 161 __lastdatatime = None
158 162 __integrationtime = None
159 163
160 164 __buffer = None
161 165
162 166 __dataReady = False
163 167
164 168 n = None
165 169
166 def __init__(self, **kwargs):
170 def __init__(self):#, **kwargs):
167 171
168 Operation.__init__(self, **kwargs)
172 Operation.__init__(self)#, **kwargs)
169 173 # self.isConfig = False
170 174
171 175 def setup(self, n=None, timeInterval=None, overlapping=False):
172 176 """
173 177 Set the parameters of the integration class.
174 178
175 179 Inputs:
176 180
177 181 n : Number of coherent integrations
178 182 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
179 183 overlapping :
180 184
181 185 """
182 186
183 187 self.__initime = None
184 188 self.__lastdatatime = 0
185 189 self.__buffer = None
186 190 self.__dataReady = False
187 191
188 192
189 193 if n == None and timeInterval == None:
190 194 raise ValueError("n or timeInterval should be specified ...")
191 195
192 196 if n != None:
193 197 self.n = n
194 198 self.__byTime = False
195 199 else:
196 200 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
197 201 self.n = 9999
198 202 self.__byTime = True
199 203
200 204 if overlapping:
201 205 self.__withOverapping = True
202 206 self.__buffer = None
203 207 else:
204 208 self.__withOverapping = False
205 209 self.__buffer = 0
206 210
207 211 self.__profIndex = 0
208 212
209 213 def putData(self, data):
210 214
211 215 """
212 216 Add a profile to the __buffer and increase in one the __profileIndex
213 217
214 218 """
215 219
216 220 if not self.__withOverapping:
217 221 self.__buffer += data.copy()
218 222 self.__profIndex += 1
219 223 return
220 224
221 225 #Overlapping data
222 226 nChannels, nHeis = data.shape
223 227 data = numpy.reshape(data, (1, nChannels, nHeis))
224 228
225 229 #If the buffer is empty then it takes the data value
226 230 if self.__buffer is None:
227 231 self.__buffer = data
228 232 self.__profIndex += 1
229 233 return
230 234
231 235 #If the buffer length is lower than n then stakcing the data value
232 236 if self.__profIndex < self.n:
233 237 self.__buffer = numpy.vstack((self.__buffer, data))
234 238 self.__profIndex += 1
235 239 return
236 240
237 241 #If the buffer length is equal to n then replacing the last buffer value with the data value
238 242 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
239 243 self.__buffer[self.n-1] = data
240 244 self.__profIndex = self.n
241 245 return
242 246
243 247
244 248 def pushData(self):
245 249 """
246 250 Return the sum of the last profiles and the profiles used in the sum.
247 251
248 252 Affected:
249 253
250 254 self.__profileIndex
251 255
252 256 """
253 257
254 258 if not self.__withOverapping:
255 259 data = self.__buffer
256 260 n = self.__profIndex
257 261
258 262 self.__buffer = 0
259 263 self.__profIndex = 0
260 264
261 265 return data, n
262 266
263 267 #Integration with Overlapping
264 268 data = numpy.sum(self.__buffer, axis=0)
265 269 n = self.__profIndex
266 270
267 271 return data, n
268 272
269 273 def byProfiles(self, data):
270 274
271 275 self.__dataReady = False
272 276 avgdata = None
273 277 # n = None
274 278
275 279 self.putData(data)
276 280
277 281 if self.__profIndex == self.n:
278 282
279 283 avgdata, n = self.pushData()
280 284 self.__dataReady = True
281 285
282 286 return avgdata
283 287
284 288 def byTime(self, data, datatime):
285 289
286 290 self.__dataReady = False
287 291 avgdata = None
288 292 n = None
289 293
290 294 self.putData(data)
291 295
292 296 if (datatime - self.__initime) >= self.__integrationtime:
293 297 avgdata, n = self.pushData()
294 298 self.n = n
295 299 self.__dataReady = True
296 300
297 301 return avgdata
298 302
299 303 def integrate(self, data, datatime=None):
300 304
301 305 if self.__initime == None:
302 306 self.__initime = datatime
303 307
304 308 if self.__byTime:
305 309 avgdata = self.byTime(data, datatime)
306 310 else:
307 311 avgdata = self.byProfiles(data)
308 312
309 313
310 314 self.__lastdatatime = datatime
311 315
312 316 if avgdata is None:
313 317 return None, None
314 318
315 319 avgdatatime = self.__initime
316 320
317 321 deltatime = datatime -self.__lastdatatime
318 322
319 323 if not self.__withOverapping:
320 324 self.__initime = datatime
321 325 else:
322 326 self.__initime += deltatime
323 327
324 328 return avgdata, avgdatatime
325 329
326 330 def run(self, dataOut, n=None, timeInterval=None, overlapping=False, **kwargs):
327 331
328 332 if not self.isConfig:
329 333 self.setup(n=n, timeInterval=timeInterval, overlapping=overlapping)
330 334 self.isConfig = True
331 335
332 336 avgdata, avgdatatime = self.integrate(dataOut.data_spc, dataOut.utctime)
333 337
334 338 # dataOut.timeInterval *= n
335 339 dataOut.flagNoData = True
336 340
337 341 if self.__dataReady:
338 342 dataOut.data_spc = avgdata
339 343 dataOut.nIncohInt *= self.n
340 344 # dataOut.nCohInt *= self.n
341 345 dataOut.utctime = avgdatatime
342 346 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nIncohInt
343 347 # dataOut.timeInterval = self.__timeInterval*self.n
344 dataOut.flagNoData = False No newline at end of file
348 dataOut.flagNoData = False
349
350 return dataOut No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now