##// END OF EJS Templates
schain-jc
RSS
Multiprocessing for Spectra (all operation) working
George Yong -
r1171:92232440951e
parent child
Show More
Side by Side Unified
Context file:
r1171:92232440951e Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1342 +1,1332
1 1 '''
2 Updated on January , 2018, for multiprocessing purposes
3 Author: Sergio Cortez
2 4 Created on September , 2012
3 @author:
4 5 '''
5
6 from platform import python_version
6 7 import sys
7 8 import ast
8 9 import datetime
9 10 import traceback
10 11 import math
11 12 import time
13 import zmq
12 14 from multiprocessing import Process, cpu_count
13 15
14 16 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 17 from xml.dom import minidom
16 18
17 import schainpy
19
18 20 from schainpy.admin import Alarm, SchainWarning
19 from schainpy.model import *
21
22 ### Temporary imports!!!
23 # from schainpy.model import *
24 from schainpy.model.io import *
25 from schainpy.model.graphics import *
26 from schainpy.model.proc.jroproc_base import *
27 from schainpy.model.proc.bltrproc_parameters import *
28 from schainpy.model.proc.jroproc_spectra import *
29 from schainpy.model.proc.jroproc_parameters import *
30 from schainpy.model.utils.jroutils_publish import *
20 31 from schainpy.utils import log
32 ###
21 33
22 34 DTYPES = {
23 35 'Voltage': '.r',
24 36 'Spectra': '.pdata'
25 37 }
26 38
27 39
28 40 def MPProject(project, n=cpu_count()):
29 41 '''
30 42 Project wrapper to run schain in n processes
31 43 '''
32 44
33 45 rconf = project.getReadUnitObj()
34 46 op = rconf.getOperationObj('run')
35 47 dt1 = op.getParameterValue('startDate')
36 48 dt2 = op.getParameterValue('endDate')
37 49 tm1 = op.getParameterValue('startTime')
38 50 tm2 = op.getParameterValue('endTime')
39 51 days = (dt2 - dt1).days
40 52
41 53 for day in range(days + 1):
42 54 skip = 0
43 55 cursor = 0
44 56 processes = []
45 57 dt = dt1 + datetime.timedelta(day)
46 58 dt_str = dt.strftime('%Y/%m/%d')
47 59 reader = JRODataReader()
48 60 paths, files = reader.searchFilesOffLine(path=rconf.path,
49 61 startDate=dt,
50 62 endDate=dt,
51 63 startTime=tm1,
52 64 endTime=tm2,
53 65 ext=DTYPES[rconf.datatype])
54 66 nFiles = len(files)
55 67 if nFiles == 0:
56 68 continue
57 69 skip = int(math.ceil(nFiles / n))
58 70 while nFiles > cursor * skip:
59 71 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
60 72 skip=skip)
61 73 p = project.clone()
62 74 p.start()
63 75 processes.append(p)
64 76 cursor += 1
65 77
66 78 def beforeExit(exctype, value, trace):
67 79 for process in processes:
68 80 process.terminate()
69 81 process.join()
70 82 print(traceback.print_tb(trace))
71 83
72 84 sys.excepthook = beforeExit
73 85
74 86 for process in processes:
75 87 process.join()
76 88 process.terminate()
77 89
78 90 time.sleep(3)
79 91
80
81 92 class ParameterConf():
82 93
83 94 id = None
84 95 name = None
85 96 value = None
86 97 format = None
87 98
88 99 __formated_value = None
89 100
90 101 ELEMENTNAME = 'Parameter'
91 102
92 103 def __init__(self):
93 104
94 105 self.format = 'str'
95 106
96 107 def getElementName(self):
97 108
98 109 return self.ELEMENTNAME
99 110
100 111 def getValue(self):
101 112
102 113 value = self.value
103 114 format = self.format
104 115
105 116 if self.__formated_value != None:
106 117
107 118 return self.__formated_value
108 119
109 120 if format == 'obj':
110 121 return value
111 122
112 123 if format == 'str':
113 124 self.__formated_value = str(value)
114 125 return self.__formated_value
115 126
116 127 if value == '':
117 128 raise ValueError('%s: This parameter value is empty' % self.name)
118 129
119 130 if format == 'list':
120 131 strList = value.split(',')
121 132
122 133 self.__formated_value = strList
123 134
124 135 return self.__formated_value
125 136
126 137 if format == 'intlist':
127 138 '''
128 139 Example:
129 140 value = (0,1,2)
130 141 '''
131 142
132 143 new_value = ast.literal_eval(value)
133 144
134 145 if type(new_value) not in (tuple, list):
135 146 new_value = [int(new_value)]
136 147
137 148 self.__formated_value = new_value
138 149
139 150 return self.__formated_value
140 151
141 152 if format == 'floatlist':
142 153 '''
143 154 Example:
144 155 value = (0.5, 1.4, 2.7)
145 156 '''
146 157
147 158 new_value = ast.literal_eval(value)
148 159
149 160 if type(new_value) not in (tuple, list):
150 161 new_value = [float(new_value)]
151 162
152 163 self.__formated_value = new_value
153 164
154 165 return self.__formated_value
155 166
156 167 if format == 'date':
157 168 strList = value.split('/')
158 169 intList = [int(x) for x in strList]
159 170 date = datetime.date(intList[0], intList[1], intList[2])
160 171
161 172 self.__formated_value = date
162 173
163 174 return self.__formated_value
164 175
165 176 if format == 'time':
166 177 strList = value.split(':')
167 178 intList = [int(x) for x in strList]
168 179 time = datetime.time(intList[0], intList[1], intList[2])
169 180
170 181 self.__formated_value = time
171 182
172 183 return self.__formated_value
173 184
174 185 if format == 'pairslist':
175 186 '''
176 187 Example:
177 188 value = (0,1),(1,2)
178 189 '''
179 190
180 191 new_value = ast.literal_eval(value)
181 192
182 193 if type(new_value) not in (tuple, list):
183 194 raise ValueError('%s has to be a tuple or list of pairs' % value)
184 195
185 196 if type(new_value[0]) not in (tuple, list):
186 197 if len(new_value) != 2:
187 198 raise ValueError('%s has to be a tuple or list of pairs' % value)
188 199 new_value = [new_value]
189 200
190 201 for thisPair in new_value:
191 202 if len(thisPair) != 2:
192 203 raise ValueError('%s has to be a tuple or list of pairs' % value)
193 204
194 205 self.__formated_value = new_value
195 206
196 207 return self.__formated_value
197 208
198 209 if format == 'multilist':
199 210 '''
200 211 Example:
201 212 value = (0,1,2),(3,4,5)
202 213 '''
203 214 multiList = ast.literal_eval(value)
204 215
205 216 if type(multiList[0]) == int:
206 217 multiList = ast.literal_eval('(' + value + ')')
207 218
208 219 self.__formated_value = multiList
209 220
210 221 return self.__formated_value
211 222
212 223 if format == 'bool':
213 224 value = int(value)
214 225
215 226 if format == 'int':
216 227 value = float(value)
217 228
218 229 format_func = eval(format)
219 230
220 231 self.__formated_value = format_func(value)
221 232
222 233 return self.__formated_value
223 234
224 235 def updateId(self, new_id):
225 236
226 237 self.id = str(new_id)
227 238
228 239 def setup(self, id, name, value, format='str'):
229 240 self.id = str(id)
230 241 self.name = name
231 242 if format == 'obj':
232 243 self.value = value
233 244 else:
234 245 self.value = str(value)
235 246 self.format = str.lower(format)
236 247
237 248 self.getValue()
238 249
239 250 return 1
240 251
241 252 def update(self, name, value, format='str'):
242 253
243 254 self.name = name
244 255 self.value = str(value)
245 256 self.format = format
246 257
247 258 def makeXml(self, opElement):
248 259 if self.name not in ('queue',):
249 260 parmElement = SubElement(opElement, self.ELEMENTNAME)
250 261 parmElement.set('id', str(self.id))
251 262 parmElement.set('name', self.name)
252 263 parmElement.set('value', self.value)
253 264 parmElement.set('format', self.format)
254 265
255 266 def readXml(self, parmElement):
256 267
257 268 self.id = parmElement.get('id')
258 269 self.name = parmElement.get('name')
259 270 self.value = parmElement.get('value')
260 271 self.format = str.lower(parmElement.get('format'))
261 272
262 273 # Compatible with old signal chain version
263 274 if self.format == 'int' and self.name == 'idfigure':
264 275 self.name = 'id'
265 276
266 277 def printattr(self):
267 278
268 279 print('Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format))
269 280
270
271 281 class OperationConf():
272 282
273 283 id = None
274 284 name = None
275 285 priority = None
276 286 type = None
277 287
278 288 parmConfObjList = []
279 289
280 290 ELEMENTNAME = 'Operation'
281 291
282 292 def __init__(self):
283 293
284 294 self.id = '0'
285 295 self.name = None
286 296 self.priority = None
287 self.type = 'self'
297 self.topic = None
288 298
289 299 def __getNewId(self):
290 300
291 301 return int(self.id) * 10 + len(self.parmConfObjList) + 1
292 302
303 def getId(self):
304 return self.id
305
293 306 def updateId(self, new_id):
294 307
295 308 self.id = str(new_id)
296 309
297 310 n = 1
298 311 for parmObj in self.parmConfObjList:
299 312
300 313 idParm = str(int(new_id) * 10 + n)
301 314 parmObj.updateId(idParm)
302 315
303 316 n += 1
304 317
305 318 def getElementName(self):
306 319
307 320 return self.ELEMENTNAME
308 321
309 322 def getParameterObjList(self):
310 323
311 324 return self.parmConfObjList
312 325
313 326 def getParameterObj(self, parameterName):
314 327
315 328 for parmConfObj in self.parmConfObjList:
316 329
317 330 if parmConfObj.name != parameterName:
318 331 continue
319 332
320 333 return parmConfObj
321 334
322 335 return None
323 336
324 337 def getParameterObjfromValue(self, parameterValue):
325 338
326 339 for parmConfObj in self.parmConfObjList:
327 340
328 341 if parmConfObj.getValue() != parameterValue:
329 342 continue
330 343
331 344 return parmConfObj.getValue()
332 345
333 346 return None
334 347
335 348 def getParameterValue(self, parameterName):
336 349
337 350 parameterObj = self.getParameterObj(parameterName)
338 351
339 352 # if not parameterObj:
340 353 # return None
341 354
342 355 value = parameterObj.getValue()
343 356
344 357 return value
345 358
346 359 def getKwargs(self):
347 360
348 361 kwargs = {}
349 362
350 363 for parmConfObj in self.parmConfObjList:
351 364 if self.name == 'run' and parmConfObj.name == 'datatype':
352 365 continue
353 366
354 367 kwargs[parmConfObj.name] = parmConfObj.getValue()
355 368
356 369 return kwargs
357 370
358 371 def setup(self, id, name, priority, type):
359 372
360 373 self.id = str(id)
361 374 self.name = name
362 375 self.type = type
363 376 self.priority = priority
364
365 377 self.parmConfObjList = []
366 378
367 379 def removeParameters(self):
368 380
369 381 for obj in self.parmConfObjList:
370 382 del obj
371 383
372 384 self.parmConfObjList = []
373 385
374 386 def addParameter(self, name, value, format='str'):
375 387
376 388 if value is None:
377 389 return None
378 390 id = self.__getNewId()
379 391
380 392 parmConfObj = ParameterConf()
381 393 if not parmConfObj.setup(id, name, value, format):
382 394 return None
383 395
384 396 self.parmConfObjList.append(parmConfObj)
385 397
386 398 return parmConfObj
387 399
388 400 def changeParameter(self, name, value, format='str'):
389 401
390 402 parmConfObj = self.getParameterObj(name)
391 403 parmConfObj.update(name, value, format)
392 404
393 405 return parmConfObj
394 406
395 407 def makeXml(self, procUnitElement):
396 408
397 409 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
398 410 opElement.set('id', str(self.id))
399 411 opElement.set('name', self.name)
400 412 opElement.set('type', self.type)
401 413 opElement.set('priority', str(self.priority))
402 414
403 415 for parmConfObj in self.parmConfObjList:
404 416 parmConfObj.makeXml(opElement)
405 417
406 418 def readXml(self, opElement):
407 419
408 420 self.id = opElement.get('id')
409 421 self.name = opElement.get('name')
410 422 self.type = opElement.get('type')
411 423 self.priority = opElement.get('priority')
412 424
413 425 # Compatible with old signal chain version
414 426 # Use of 'run' method instead 'init'
415 427 if self.type == 'self' and self.name == 'init':
416 428 self.name = 'run'
417 429
418 430 self.parmConfObjList = []
419 431
420 432 parmElementList = opElement.iter(ParameterConf().getElementName())
421 433
422 434 for parmElement in parmElementList:
423 435 parmConfObj = ParameterConf()
424 436 parmConfObj.readXml(parmElement)
425 437
426 438 # Compatible with old signal chain version
427 439 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
428 440 if self.type != 'self' and self.name == 'Plot':
429 441 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
430 442 self.name = parmConfObj.value
431 443 continue
432 444
433 445 self.parmConfObjList.append(parmConfObj)
434 446
435 447 def printattr(self):
436 448
437 449 print('%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
438 450 self.id,
439 451 self.name,
440 452 self.type,
441 453 self.priority))
442 454
443 455 for parmConfObj in self.parmConfObjList:
444 456 parmConfObj.printattr()
445 457
446 def createObject(self, plotter_queue=None):
447
448 if self.type == 'self':
449 raise ValueError('This operation type cannot be created')
458 def createObject(self):
450 459
451 if self.type == 'plotter':
452 if not plotter_queue:
453 raise ValueError('plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)')
454
455 opObj = Plotter(self.name, plotter_queue)
460 className = eval(self.name)
461 kwargs = self.getKwargs()
456 462
457 if self.type == 'external' or self.type == 'other':
463 opObj = className(self.id, **kwargs)
458 464
459 className = eval(self.name)
460 kwargs = self.getKwargs()
465 opObj.start()
461 466
462 opObj = className(**kwargs)
467 print(' Operation created')
463 468
464 469 return opObj
465 470
466
467 471 class ProcUnitConf():
468 472
469 473 id = None
470 474 name = None
471 475 datatype = None
472 476 inputId = None
473 477 parentId = None
474 478
475 479 opConfObjList = []
476 480
477 481 procUnitObj = None
478 482 opObjList = []
479 483
480 484 ELEMENTNAME = 'ProcUnit'
481 485
482 486 def __init__(self):
483 487
484 488 self.id = None
485 489 self.datatype = None
486 490 self.name = None
487 self.inputId = None
491 self.inputId = None
488 492
489 493 self.opConfObjList = []
490 494
491 495 self.procUnitObj = None
492 496 self.opObjDict = {}
493 497
494 498 def __getPriority(self):
495 499
496 500 return len(self.opConfObjList) + 1
497 501
498 502 def __getNewId(self):
499 503
500 504 return int(self.id) * 10 + len(self.opConfObjList) + 1
501 505
502 506 def getElementName(self):
503 507
504 508 return self.ELEMENTNAME
505 509
506 510 def getId(self):
507 511
508 512 return self.id
509 513
510 def updateId(self, new_id, parentId=parentId):
511
514 def updateId(self, new_id, parentId=parentId):
515 '''
512 516 new_id = int(parentId) * 10 + (int(self.id) % 10)
513 517 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
514 518
515 519 # If this proc unit has not inputs
516 if self.inputId == '0':
517 new_inputId = 0
520 #if self.inputId == '0':
521 #new_inputId = 0
518 522
519 523 n = 1
520 524 for opConfObj in self.opConfObjList:
521 525
522 526 idOp = str(int(new_id) * 10 + n)
523 527 opConfObj.updateId(idOp)
524 528
525 529 n += 1
526 530
527 531 self.parentId = str(parentId)
528 532 self.id = str(new_id)
529 self.inputId = str(new_inputId)
530
533 #self.inputId = str(new_inputId)
534 '''
535 n = 1
531 536 def getInputId(self):
532 537
533 538 return self.inputId
534 539
535 540 def getOperationObjList(self):
536 541
537 542 return self.opConfObjList
538 543
539 544 def getOperationObj(self, name=None):
540 545
541 546 for opConfObj in self.opConfObjList:
542 547
543 548 if opConfObj.name != name:
544 549 continue
545 550
546 551 return opConfObj
547 552
548 553 return None
549 554
550 555 def getOpObjfromParamValue(self, value=None):
551 556
552 557 for opConfObj in self.opConfObjList:
553 558 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 559 continue
555 560 return opConfObj
556 561 return None
557 562
558 563 def getProcUnitObj(self):
559 564
560 565 return self.procUnitObj
561 566
562 567 def setup(self, id, name, datatype, inputId, parentId=None):
563
568 '''
569 id sera el topico a publicar
570 inputId sera el topico a subscribirse
571 '''
572
564 573 # Compatible with old signal chain version
565 574 if datatype == None and name == None:
566 575 raise ValueError('datatype or name should be defined')
567 576
577 #Definir una condicion para inputId cuando sea 0
578
568 579 if name == None:
569 580 if 'Proc' in datatype:
570 581 name = datatype
571 582 else:
572 583 name = '%sProc' % (datatype)
573 584
574 585 if datatype == None:
575 586 datatype = name.replace('Proc', '')
576 587
577 588 self.id = str(id)
578 589 self.name = name
579 590 self.datatype = datatype
580 self.inputId = inputId
591 self.inputId = inputId
581 592 self.parentId = parentId
582
583 593 self.opConfObjList = []
584 594
585 self.addOperation(name='run', optype='self')
595 self.addOperation(name='run', optype='self')
586 596
587 597 def removeOperations(self):
588 598
589 599 for obj in self.opConfObjList:
590 600 del obj
591 601
592 602 self.opConfObjList = []
593 603 self.addOperation(name='run')
594 604
595 605 def addParameter(self, **kwargs):
596 606 '''
597 607 Add parameters to 'run' operation
598 608 '''
599 609 opObj = self.opConfObjList[0]
600 610
601 611 opObj.addParameter(**kwargs)
602 612
603 613 return opObj
604 614
605 def addOperation(self, name, optype='self'):
615 def addOperation(self, name, optype = 'self'):
616 '''
617 Actualizacion - > proceso comunicacion
618 En el caso de optype='self', elminar. DEfinir comuncacion IPC -> Topic
619 definir el tipoc de socket o comunicacion ipc++
620
621 '''
606 622
607 623 id = self.__getNewId()
608 priority = self.__getPriority()
624 priority = self.__getPriority() # Sin mucho sentido, pero puede usarse
609 625
610 626 opConfObj = OperationConf()
611 627 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 628
613 629 self.opConfObjList.append(opConfObj)
614 630
615 631 return opConfObj
616 632
617 633 def makeXml(self, projectElement):
618 634
619 635 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 636 procUnitElement.set('id', str(self.id))
621 637 procUnitElement.set('name', self.name)
622 638 procUnitElement.set('datatype', self.datatype)
623 639 procUnitElement.set('inputId', str(self.inputId))
624 640
625 641 for opConfObj in self.opConfObjList:
626 642 opConfObj.makeXml(procUnitElement)
627 643
628 644 def readXml(self, upElement):
629 645
630 646 self.id = upElement.get('id')
631 647 self.name = upElement.get('name')
632 648 self.datatype = upElement.get('datatype')
633 649 self.inputId = upElement.get('inputId')
634 650
635 651 if self.ELEMENTNAME == 'ReadUnit':
636 652 self.datatype = self.datatype.replace('Reader', '')
637 653
638 654 if self.ELEMENTNAME == 'ProcUnit':
639 655 self.datatype = self.datatype.replace('Proc', '')
640 656
641 657 if self.inputId == 'None':
642 658 self.inputId = '0'
643 659
644 660 self.opConfObjList = []
645 661
646 662 opElementList = upElement.iter(OperationConf().getElementName())
647 663
648 664 for opElement in opElementList:
649 665 opConfObj = OperationConf()
650 666 opConfObj.readXml(opElement)
651 667 self.opConfObjList.append(opConfObj)
652 668
653 669 def printattr(self):
654 670
655 671 print('%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
656 672 self.id,
657 673 self.name,
658 674 self.datatype,
659 675 self.inputId))
660 676
661 677 for opConfObj in self.opConfObjList:
662 678 opConfObj.printattr()
663 679
664 680 def getKwargs(self):
665 681
666 682 opObj = self.opConfObjList[0]
667 683 kwargs = opObj.getKwargs()
668 684
669 685 return kwargs
670 686
671 def createObjects(self, plotter_queue=None):
687 def createObjects(self, dictUnits):
688 '''
689 Instancia de unidades de procesamiento.
672 690
691 '''
673 692 className = eval(self.name)
674 693 kwargs = self.getKwargs()
675 procUnitObj = className(**kwargs)
694 procUnitObj = className(self.id, self.inputId, dictUnits, **kwargs) # necesitan saber su id y su entrada por fines de ipc
695
676 696
677 697 for opConfObj in self.opConfObjList:
678 698
679 699 if opConfObj.type == 'self' and self.name == 'run':
680 700 continue
681 701 elif opConfObj.type == 'self':
682 702 procUnitObj.addOperationKwargs(
683 703 opConfObj.id, **opConfObj.getKwargs())
684 704 continue
685
686 opObj = opConfObj.createObject(plotter_queue)
687
688 self.opObjDict[opConfObj.id] = opObj
689
690 procUnitObj.addOperation(opObj, opConfObj.id)
705 print("Creating operation process:", opConfObj.name, "for", self.name)
706 opObj = opConfObj.createObject()
707
708
709 #self.opObjDict[opConfObj.id] = opObj.name
710
711 procUnitObj.addOperation(opConfObj.name, opConfObj.id)
712
713 procUnitObj.start()
691 714
692 715 self.procUnitObj = procUnitObj
716
693 717
694 718 return procUnitObj
695 719
696 720 def run(self):
697
698 is_ok = False
699
721
722 is_ok = True
723 """
700 724 for opConfObj in self.opConfObjList:
701 725
702 726 kwargs = {}
703 727 for parmConfObj in opConfObj.getParameterObjList():
704 728 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 729 continue
706 730
707 731 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 732
709 733 sts = self.procUnitObj.call(opType=opConfObj.type,
710 734 opName=opConfObj.name,
711 735 opId=opConfObj.id)
712 736
713 737 is_ok = is_ok or sts
714
738
739 """
715 740 return is_ok
716
741
742
717 743 def close(self):
718 744
719 745 for opConfObj in self.opConfObjList:
720 746 if opConfObj.type == 'self':
721 747 continue
722 748
723 749 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
724 750 opObj.close()
725 751
726 752 self.procUnitObj.close()
727 753
728 754 return
729 755
730 756
731 757 class ReadUnitConf(ProcUnitConf):
732 758
733 759 path = None
734 760 startDate = None
735 761 endDate = None
736 762 startTime = None
737 763 endTime = None
738 764
739 765 ELEMENTNAME = 'ReadUnit'
740 766
741 767 def __init__(self):
742 768
743 769 self.id = None
744 770 self.datatype = None
745 771 self.name = None
746 772 self.inputId = None
747 773
748 774 self.parentId = None
749 775
750 776 self.opConfObjList = []
751 777 self.opObjList = []
752 778
753 779 def getElementName(self):
754 780
755 return self.ELEMENTNAME
756
781 return self.ELEMENTNAME
782
757 783 def setup(self, id, name, datatype, path='', startDate='', endDate='',
758 784 startTime='', endTime='', parentId=None, server=None, **kwargs):
759 785
786
787 '''
788 *****el id del proceso sera el Topico
789
790 Adicion de {topic}, si no esta presente -> error
791 kwargs deben ser trasmitidos en la instanciacion
792
793 '''
794
760 795 # Compatible with old signal chain version
761 796 if datatype == None and name == None:
762 797 raise ValueError('datatype or name should be defined')
763 798 if name == None:
764 799 if 'Reader' in datatype:
765 800 name = datatype
766 801 datatype = name.replace('Reader','')
767 802 else:
768 803 name = '{}Reader'.format(datatype)
769 804 if datatype == None:
770 805 if 'Reader' in name:
771 806 datatype = name.replace('Reader','')
772 807 else:
773 808 datatype = name
774 809 name = '{}Reader'.format(name)
775 810
776 811 self.id = id
777 812 self.name = name
778 813 self.datatype = datatype
779 814 if path != '':
780 815 self.path = os.path.abspath(path)
781 816 self.startDate = startDate
782 817 self.endDate = endDate
783 818 self.startTime = startTime
784 819 self.endTime = endTime
785 820 self.inputId = '0'
786 821 self.parentId = parentId
787 822 self.server = server
788 823 self.addRunOperation(**kwargs)
789 824
790 825 def update(self, **kwargs):
791 826
792 827 if 'datatype' in kwargs:
793 828 datatype = kwargs.pop('datatype')
794 829 if 'Reader' in datatype:
795 830 self.name = datatype
796 831 else:
797 832 self.name = '%sReader' % (datatype)
798 833 self.datatype = self.name.replace('Reader', '')
799 834
800 835 attrs = ('path', 'startDate', 'endDate',
801 836 'startTime', 'endTime', 'parentId')
802 837
803 838 for attr in attrs:
804 839 if attr in kwargs:
805 840 setattr(self, attr, kwargs.pop(attr))
806 841
807 842 self.inputId = '0'
808 843 self.updateRunOperation(**kwargs)
809 844
810 845 def removeOperations(self):
811 846
812 847 for obj in self.opConfObjList:
813 848 del obj
814 849
815 850 self.opConfObjList = []
816 851
817 def addRunOperation(self, **kwargs):
852 def addRunOperation(self, **kwargs):
818 853
819 opObj = self.addOperation(name='run', optype='self')
854 opObj = self.addOperation(name='run', optype='self')
820 855
821 856 if self.server is None:
822 857 opObj.addParameter(
823 858 name='datatype', value=self.datatype, format='str')
824 859 opObj.addParameter(name='path', value=self.path, format='str')
825 860 opObj.addParameter(
826 861 name='startDate', value=self.startDate, format='date')
827 862 opObj.addParameter(
828 863 name='endDate', value=self.endDate, format='date')
829 864 opObj.addParameter(
830 865 name='startTime', value=self.startTime, format='time')
831 866 opObj.addParameter(
832 867 name='endTime', value=self.endTime, format='time')
833 868
834 869 for key, value in list(kwargs.items()):
835 870 opObj.addParameter(name=key, value=value,
836 871 format=type(value).__name__)
837 872 else:
838 873 opObj.addParameter(name='server', value=self.server, format='str')
839 874
840 875 return opObj
841 876
842 877 def updateRunOperation(self, **kwargs):
843 878
844 879 opObj = self.getOperationObj(name='run')
845 880 opObj.removeParameters()
846 881
847 882 opObj.addParameter(name='datatype', value=self.datatype, format='str')
848 883 opObj.addParameter(name='path', value=self.path, format='str')
849 884 opObj.addParameter(
850 885 name='startDate', value=self.startDate, format='date')
851 886 opObj.addParameter(name='endDate', value=self.endDate, format='date')
852 887 opObj.addParameter(
853 888 name='startTime', value=self.startTime, format='time')
854 889 opObj.addParameter(name='endTime', value=self.endTime, format='time')
855 890
856 891 for key, value in list(kwargs.items()):
857 892 opObj.addParameter(name=key, value=value,
858 893 format=type(value).__name__)
859 894
860 895 return opObj
861 896
862 897 def readXml(self, upElement):
863 898
864 899 self.id = upElement.get('id')
865 900 self.name = upElement.get('name')
866 901 self.datatype = upElement.get('datatype')
867 902 self.inputId = upElement.get('inputId')
868 903
869 904 if self.ELEMENTNAME == 'ReadUnit':
870 905 self.datatype = self.datatype.replace('Reader', '')
871 906
872 907 if self.inputId == 'None':
873 908 self.inputId = '0'
874 909
875 910 self.opConfObjList = []
876 911
877 912 opElementList = upElement.iter(OperationConf().getElementName())
878 913
879 914 for opElement in opElementList:
880 915 opConfObj = OperationConf()
881 916 opConfObj.readXml(opElement)
882 917 self.opConfObjList.append(opConfObj)
883 918
884 919 if opConfObj.name == 'run':
885 920 self.path = opConfObj.getParameterValue('path')
886 921 self.startDate = opConfObj.getParameterValue('startDate')
887 922 self.endDate = opConfObj.getParameterValue('endDate')
888 923 self.startTime = opConfObj.getParameterValue('startTime')
889 924 self.endTime = opConfObj.getParameterValue('endTime')
890 925
891 926
892 927 class Project(Process):
893 928
894 929 id = None
895 # name = None
896 930 description = None
897 931 filename = None
898 932
899 933 procUnitConfObjDict = None
900 934
901 935 ELEMENTNAME = 'Project'
902 936
903 plotterQueue = None
937
904 938
905 def __init__(self, plotter_queue=None):
939 def __init__(self):
906 940
907 941 Process.__init__(self)
908 self.id = None
942 self.id = None
909 943 self.description = None
910 944 self.email = None
911 945 self.alarm = None
912 self.plotterQueue = plotter_queue
913 946 self.procUnitConfObjDict = {}
914 947
915 948 def __getNewId(self):
916 949
917 950 idList = list(self.procUnitConfObjDict.keys())
918 951
919 952 id = int(self.id) * 10
920 953
921 954 while True:
922 955 id += 1
923 956
924 957 if str(id) in idList:
925 958 continue
926 959
927 960 break
928 961
929 962 return str(id)
930 963
931 964 def getElementName(self):
932 965
933 966 return self.ELEMENTNAME
934 967
935 968 def getId(self):
936 969
937 970 return self.id
938 971
939 972 def updateId(self, new_id):
940 973
941 974 self.id = str(new_id)
942 975
943 976 keyList = list(self.procUnitConfObjDict.keys())
944 977 keyList.sort()
945 978
946 979 n = 1
947 980 newProcUnitConfObjDict = {}
948 981
949 982 for procKey in keyList:
950 983
951 984 procUnitConfObj = self.procUnitConfObjDict[procKey]
952 985 idProcUnit = str(int(self.id) * 10 + n)
953 986 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
954 987 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
955 988 n += 1
956 989
957 990 self.procUnitConfObjDict = newProcUnitConfObjDict
958 991
959 992 def setup(self, id, name='', description='', email=None, alarm=[]):
960 993
961 print()
994 print(' ')
962 995 print('*' * 60)
963 print(' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__)
996 print('* Starting SIGNAL CHAIN PROCESSING (Multiprocessing) v%s *' % schainpy.__version__)
964 997 print('*' * 60)
965 print()
998 print("* Python " + python_version() + " *")
999 print('*' * 19)
1000 print(' ')
966 1001 self.id = str(id)
967 self.description = description
1002 self.description = description
968 1003 self.email = email
969 1004 self.alarm = alarm
970 1005
971 1006 def update(self, **kwargs):
972 1007
973 1008 for key, value in list(kwargs.items()):
974 1009 setattr(self, key, value)
975 1010
976 1011 def clone(self):
977 1012
978 1013 p = Project()
979 1014 p.procUnitConfObjDict = self.procUnitConfObjDict
980 1015 return p
981 1016
982 1017 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
983 1018
1019 '''
1020 Actualizacion:
1021 Se agrego un nuevo argumento: topic -relativo a la forma de comunicar los procesos simultaneos
1022
1023 * El id del proceso sera el topico al que se deben subscribir los procUnits para recibir la informacion(data)
1024
1025 '''
1026
984 1027 if id is None:
985 1028 idReadUnit = self.__getNewId()
986 1029 else:
987 1030 idReadUnit = str(id)
988 1031
989 1032 readUnitConfObj = ReadUnitConf()
990 1033 readUnitConfObj.setup(idReadUnit, name, datatype,
991 1034 parentId=self.id, **kwargs)
992 1035
993 1036 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
994
1037
995 1038 return readUnitConfObj
996 1039
997 1040 def addProcUnit(self, inputId='0', datatype=None, name=None):
998 1041
999 idProcUnit = self.__getNewId()
1042 '''
1043 Actualizacion:
1044 Se agrego dos nuevos argumentos: topic_read (lee data de otro procUnit) y topic_write(escribe o envia data a otro procUnit)
1045 Deberia reemplazar a "inputId"
1046
1047 ** A fin de mantener el inputID, este sera la representaacion del topicoal que deben subscribirse. El ID propio de la intancia
1048 (proceso) sera el topico de la publicacion, todo sera asignado de manera dinamica.
1049
1050 '''
1051
1052 idProcUnit = self.__getNewId() #Topico para subscripcion
1000 1053
1001 1054 procUnitConfObj = ProcUnitConf()
1002 procUnitConfObj.setup(idProcUnit, name, datatype,
1003 inputId, parentId=self.id)
1055 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, #topic_read, topic_write,
1056 parentId=self.id)
1004 1057
1005 1058 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 1059
1007 1060 return procUnitConfObj
1008 1061
1009 1062 def removeProcUnit(self, id):
1010 1063
1011 1064 if id in list(self.procUnitConfObjDict.keys()):
1012 1065 self.procUnitConfObjDict.pop(id)
1013 1066
1014 1067 def getReadUnitId(self):
1015 1068
1016 1069 readUnitConfObj = self.getReadUnitObj()
1017 1070
1018 1071 return readUnitConfObj.id
1019 1072
1020 1073 def getReadUnitObj(self):
1021 1074
1022 1075 for obj in list(self.procUnitConfObjDict.values()):
1023 1076 if obj.getElementName() == 'ReadUnit':
1024 1077 return obj
1025 1078
1026 1079 return None
1027 1080
1028 1081 def getProcUnitObj(self, id=None, name=None):
1029 1082
1030 1083 if id != None:
1031 1084 return self.procUnitConfObjDict[id]
1032 1085
1033 1086 if name != None:
1034 1087 return self.getProcUnitObjByName(name)
1035 1088
1036 1089 return None
1037 1090
1038 1091 def getProcUnitObjByName(self, name):
1039 1092
1040 1093 for obj in list(self.procUnitConfObjDict.values()):
1041 1094 if obj.name == name:
1042 1095 return obj
1043 1096
1044 1097 return None
1045 1098
1046 1099 def procUnitItems(self):
1047 1100
1048 1101 return list(self.procUnitConfObjDict.items())
1049 1102
1050 1103 def makeXml(self):
1051 1104
1052 1105 projectElement = Element('Project')
1053 1106 projectElement.set('id', str(self.id))
1054 1107 projectElement.set('name', self.name)
1055 1108 projectElement.set('description', self.description)
1056 1109
1057 1110 for procUnitConfObj in list(self.procUnitConfObjDict.values()):
1058 1111 procUnitConfObj.makeXml(projectElement)
1059 1112
1060 1113 self.projectElement = projectElement
1061 1114
1062 1115 def writeXml(self, filename=None):
1063 1116
1064 1117 if filename == None:
1065 1118 if self.filename:
1066 1119 filename = self.filename
1067 1120 else:
1068 1121 filename = 'schain.xml'
1069 1122
1070 1123 if not filename:
1071 1124 print('filename has not been defined. Use setFilename(filename) for do it.')
1072 1125 return 0
1073 1126
1074 1127 abs_file = os.path.abspath(filename)
1075 1128
1076 1129 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 1130 print('No write permission on %s' % os.path.dirname(abs_file))
1078 1131 return 0
1079 1132
1080 1133 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 1134 print('File %s already exists and it could not be overwriten' % abs_file)
1082 1135 return 0
1083 1136
1084 1137 self.makeXml()
1085 1138
1086 1139 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 1140
1088 1141 self.filename = abs_file
1089 1142
1090 1143 return 1
1091 1144
1092 1145 def readXml(self, filename=None):
1093 1146
1094 1147 if not filename:
1095 1148 print('filename is not defined')
1096 1149 return 0
1097 1150
1098 1151 abs_file = os.path.abspath(filename)
1099 1152
1100 1153 if not os.path.isfile(abs_file):
1101 1154 print('%s file does not exist' % abs_file)
1102 1155 return 0
1103 1156
1104 1157 self.projectElement = None
1105 1158 self.procUnitConfObjDict = {}
1106 1159
1107 1160 try:
1108 1161 self.projectElement = ElementTree().parse(abs_file)
1109 1162 except:
1110 1163 print('Error reading %s, verify file format' % filename)
1111 1164 return 0
1112 1165
1113 1166 self.project = self.projectElement.tag
1114 1167
1115 1168 self.id = self.projectElement.get('id')
1116 1169 self.name = self.projectElement.get('name')
1117 1170 self.description = self.projectElement.get('description')
1118 1171
1119 1172 readUnitElementList = self.projectElement.iter(
1120 1173 ReadUnitConf().getElementName())
1121 1174
1122 1175 for readUnitElement in readUnitElementList:
1123 1176 readUnitConfObj = ReadUnitConf()
1124 1177 readUnitConfObj.readXml(readUnitElement)
1125 1178
1126 1179 if readUnitConfObj.parentId == None:
1127 1180 readUnitConfObj.parentId = self.id
1128 1181
1129 1182 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1130 1183
1131 1184 procUnitElementList = self.projectElement.iter(
1132 1185 ProcUnitConf().getElementName())
1133 1186
1134 1187 for procUnitElement in procUnitElementList:
1135 1188 procUnitConfObj = ProcUnitConf()
1136 1189 procUnitConfObj.readXml(procUnitElement)
1137 1190
1138 1191 if procUnitConfObj.parentId == None:
1139 1192 procUnitConfObj.parentId = self.id
1140 1193
1141 1194 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1142 1195
1143 1196 self.filename = abs_file
1144 1197
1145 1198 return 1
1146 1199
1147 1200 def printattr(self):
1148 1201
1149 1202 print('Project[%s]: name = %s, description = %s' % (self.id,
1150 1203 self.name,
1151 1204 self.description))
1152 1205
1153 1206 for procUnitConfObj in list(self.procUnitConfObjDict.values()):
1154 1207 procUnitConfObj.printattr()
1155 1208
1156 1209 def createObjects(self):
1157 1210
1158 1211 for procUnitConfObj in list(self.procUnitConfObjDict.values()):
1159 procUnitConfObj.createObjects(self.plotterQueue)
1160
1161 def __connect(self, objIN, thisObj):
1162
1163 thisObj.setInput(objIN.getOutputObj())
1164
1165 def connectObjects(self):
1166
1167 for thisPUConfObj in list(self.procUnitConfObjDict.values()):
1212 print("Creating process:", procUnitConfObj.name)
1213 procUnitConfObj.createObjects(self.procUnitConfObjDict)
1214
1168 1215
1169 inputId = thisPUConfObj.getInputId()
1170
1171 if int(inputId) == 0:
1172 continue
1173
1174 # Get input object
1175 puConfINObj = self.procUnitConfObjDict[inputId]
1176 puObjIN = puConfINObj.getProcUnitObj()
1177
1178 # Get current object
1179 thisPUObj = thisPUConfObj.getProcUnitObj()
1180
1181 self.__connect(puObjIN, thisPUObj)
1216 print('All processes were created')
1182 1217
1183 1218 def __handleError(self, procUnitConfObj, modes=None, stdout=True):
1184 1219
1185 1220 import socket
1186 1221
1187 1222 if modes is None:
1188 1223 modes = self.alarm
1189 1224
1190 1225 if not self.alarm:
1191 1226 modes = []
1192 1227
1193 1228 err = traceback.format_exception(sys.exc_info()[0],
1194 1229 sys.exc_info()[1],
1195 1230 sys.exc_info()[2])
1196
1231
1197 1232 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1198 1233
1199 1234 message = ''.join(err)
1200 1235
1201 1236 if stdout:
1202 1237 sys.stderr.write(message)
1203 1238
1204 1239 subject = 'SChain v%s: Error running %s\n' % (
1205 1240 schainpy.__version__, procUnitConfObj.name)
1206 1241
1207 1242 subtitle = '%s: %s\n' % (
1208 1243 procUnitConfObj.getElementName(), procUnitConfObj.name)
1209 1244 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1210 1245 socket.gethostname())
1211 1246 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1212 1247 subtitle += 'Configuration file: %s\n' % self.filename
1213 1248 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1214 1249
1215 1250 readUnitConfObj = self.getReadUnitObj()
1216 1251 if readUnitConfObj:
1217 1252 subtitle += '\nInput parameters:\n'
1218 1253 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1219 1254 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1220 1255 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1221 1256 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1222 1257 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1223 1258 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1224 1259
1225 1260 a = Alarm(
1226 1261 modes=modes,
1227 1262 email=self.email,
1228 1263 message=message,
1229 1264 subject=subject,
1230 1265 subtitle=subtitle,
1231 1266 filename=self.filename
1232 1267 )
1233 1268
1234 1269 return a
1235 1270
1236 1271 def isPaused(self):
1237 1272 return 0
1238 1273
1239 1274 def isStopped(self):
1240 1275 return 0
1241 1276
1242 1277 def runController(self):
1243 1278 '''
1244 1279 returns 0 when this process has been stopped, 1 otherwise
1245 1280 '''
1246 1281
1247 1282 if self.isPaused():
1248 1283 print('Process suspended')
1249 1284
1250 1285 while True:
1251 1286 time.sleep(0.1)
1252 1287
1253 1288 if not self.isPaused():
1254 1289 break
1255 1290
1256 1291 if self.isStopped():
1257 1292 break
1258 1293
1259 1294 print('Process reinitialized')
1260 1295
1261 1296 if self.isStopped():
1262 1297 print('Process stopped')
1263 1298 return 0
1264 1299
1265 1300 return 1
1266 1301
1267 1302 def setFilename(self, filename):
1268 1303
1269 1304 self.filename = filename
1270 1305
1271 def setPlotterQueue(self, plotter_queue):
1272
1273 raise NotImplementedError('Use schainpy.controller_api.ControllerThread instead Project class')
1274
1275 def getPlotterQueue(self):
1276
1277 raise NotImplementedError('Use schainpy.controller_api.ControllerThread instead Project class')
1278
1279 def useExternalPlotter(self):
1306 def setProxyCom(self):
1307
1308 ctx = zmq.Context()
1309 if not os.path.exists('/tmp/socketTmp'): os.mkdir('/tmp/socketTmp')
1310 xsub = ctx.socket(zmq.XSUB)
1311 xsub.bind('ipc:///tmp/socketTmp/a')
1312 xpub = ctx.socket(zmq.XPUB)
1313 xpub.bind('ipc:///tmp/socketTmp/b')
1314
1315 print("Controller Ready: Processes and proxy created")
1316 zmq.proxy(xsub, xpub)
1280 1317
1281 raise NotImplementedError('Use schainpy.controller_api.ControllerThread instead Project class')
1318
1282 1319
1283 1320 def run(self):
1284 1321
1285 1322 log.success('Starting {}'.format(self.name), tag='')
1286 1323 self.start_time = time.time()
1287 self.createObjects()
1288 self.connectObjects()
1289
1290 keyList = list(self.procUnitConfObjDict.keys())
1291 keyList.sort()
1292
1293 err = None
1294
1295 while(True):
1296
1297 is_ok = False
1324 self.createObjects()
1325 self.setProxyCom()
1298 1326
1299 for procKey in keyList:
1300
1301 procUnitConfObj = self.procUnitConfObjDict[procKey]
1302
1303 try:
1304 sts = procUnitConfObj.run()
1305 is_ok = is_ok or sts
1306 except SchainWarning:
1307 err = self.__handleError(procUnitConfObj, modes=[2, 3], stdout=False)
1308 is_ok = False
1309 break
1310 except KeyboardInterrupt:
1311 is_ok = False
1312 break
1313 except ValueError as e:
1314 time.sleep(0.5)
1315 err = self.__handleError(procUnitConfObj)
1316 is_ok = False
1317 break
1318 except:
1319 time.sleep(0.5)
1320 err = self.__handleError(procUnitConfObj)
1321 is_ok = False
1322 break
1323
1324 # If every process unit finished so end process
1325 if not(is_ok):
1326 break
1327
1328 if not self.runController():
1329 break
1327 # Iniciar todos los procesos .start(), monitoreo de procesos. ELiminar lo de abajo
1330 1328
1331 1329 # Closing every process
1332 for procKey in keyList:
1333 procUnitConfObj = self.procUnitConfObjDict[procKey]
1334 procUnitConfObj.close()
1335
1336 if err is not None:
1337 err.start()
1338 # err.join()
1339
1340 1330 log.success('{} finished (time: {}s)'.format(
1341 1331 self.name,
1342 1332 time.time()-self.start_time)) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,657 +1,658
1 1 import os
2 2 import numpy
3 3 import time, datetime
4 4 from schainpy.model.graphics import mpldriver
5 5
6 from schainpy.model.proc.jroproc_base import Operation
6 from schainpy.model.proc.jroproc_base import MPDecorator, Operation
7
7 8
8 9 def isTimeInHourRange(datatime, xmin, xmax):
9 10
10 11 if xmin == None or xmax == None:
11 12 return 1
12 13 hour = datatime.hour + datatime.minute/60.0
13 14
14 15 if xmin < (xmax % 24):
15 16
16 17 if hour >= xmin and hour <= xmax:
17 18 return 1
18 19 else:
19 20 return 0
20 21
21 22 else:
22 23
23 24 if hour >= xmin or hour <= (xmax % 24):
24 25 return 1
25 26 else:
26 27 return 0
27 28
28 29 return 0
29 30
30 31 def isRealtime(utcdatatime):
31 32
32 33 utcnow = time.mktime(time.localtime())
33 34 delta = abs(utcnow - utcdatatime) # abs
34 35 if delta >= 30.:
35 36 return False
36 37 return True
37 38
38 39 class Figure(Operation):
39 40
40 41 __driver = mpldriver
41 42 fig = None
42 43
43 44 id = None
44 45 wintitle = None
45 46 width = None
46 47 height = None
47 48 nplots = None
48 49 timerange = None
49 50
50 51 axesObjList = []
51 52
52 53 WIDTH = 300
53 54 HEIGHT = 200
54 55 PREFIX = 'fig'
55 56
56 57 xmin = None
57 58 xmax = None
58 59
59 60 counter_imagwr = 0
60 61
61 62 figfile = None
62 63
63 64 created = False
64 65 parameters = {}
65 def __init__(self, **kwargs):
66 def __init__(self):#, **kwargs):
66 67
67 Operation.__init__(self, **kwargs)
68 Operation.__init__(self)#, **kwargs)
68 69
69 70 def __del__(self):
70 71
71 72 self.__driver.closeFigure()
72 73
73 74 def getFilename(self, name, ext='.png'):
74 75
75 76 path = '%s%03d' %(self.PREFIX, self.id)
76 77 filename = '%s_%s%s' %(self.PREFIX, name, ext)
77 78 return os.path.join(path, filename)
78 79
79 80 def getAxesObjList(self):
80 81
81 82 return self.axesObjList
82 83
83 84 def getSubplots(self):
84 85
85 86 raise NotImplementedError
86 87
87 88 def getScreenDim(self, widthplot, heightplot):
88 89
89 90 nrow, ncol = self.getSubplots()
90 91
91 92 widthscreen = widthplot*ncol
92 93 heightscreen = heightplot*nrow
93 94
94 95 return widthscreen, heightscreen
95 96
96 97 def getTimeLim(self, x, xmin=None, xmax=None, timerange=None):
97 98
98 99 # if self.xmin != None and self.xmax != None:
99 100 # if timerange == None:
100 101 # timerange = self.xmax - self.xmin
101 102 # xmin = self.xmin + timerange
102 103 # xmax = self.xmax + timerange
103 104 #
104 105 # return xmin, xmax
105 106
106 107 if timerange == None and (xmin==None or xmax==None):
107 108 timerange = 14400 #seconds
108 109
109 110 if timerange != None:
110 111 txmin = x[0] #- x[0] % min(timerange/10, 10*60)
111 112 else:
112 113 txmin = x[0] #- x[0] % 10*60
113 114
114 115 thisdatetime = datetime.datetime.utcfromtimestamp(txmin)
115 116 thisdate = datetime.datetime.combine(thisdatetime.date(), datetime.time(0,0,0))
116 117
117 118 if timerange != None:
118 119 xmin = (thisdatetime - thisdate).seconds/(60*60.)
119 120 xmax = xmin + timerange/(60*60.)
120 121
121 122 d1970 = datetime.datetime(1970,1,1)
122 123
123 124 mindt = thisdate + datetime.timedelta(hours=xmin) #- datetime.timedelta(seconds=time.timezone)
124 125 xmin_sec = (mindt - d1970).total_seconds() #time.mktime(mindt.timetuple()) - time.timezone
125 126
126 127 maxdt = thisdate + datetime.timedelta(hours=xmax) #- datetime.timedelta(seconds=time.timezone)
127 128 xmax_sec = (maxdt - d1970).total_seconds() #time.mktime(maxdt.timetuple()) - time.timezone
128 129
129 130 return xmin_sec, xmax_sec
130 131
131 132 def init(self, id, nplots, wintitle):
132 133
133 134 raise NotImplementedError("This method has been replaced by createFigure")
134 135
135 136 def createFigure(self, id, wintitle, widthplot=None, heightplot=None, show=True):
136 137
137 138 """
138 139 Crea la figura de acuerdo al driver y parametros seleccionados seleccionados.
139 140 Las dimensiones de la pantalla es calculada a partir de los atributos self.WIDTH
140 141 y self.HEIGHT y el numero de subplots (nrow, ncol)
141 142
142 143 Input:
143 144 id : Los parametros necesarios son
144 145 wintitle :
145 146
146 147 """
147 148
148 149 if widthplot == None:
149 150 widthplot = self.WIDTH
150 151
151 152 if heightplot == None:
152 153 heightplot = self.HEIGHT
153 154
154 155 self.id = id
155 156
156 157 self.wintitle = wintitle
157 158
158 159 self.widthscreen, self.heightscreen = self.getScreenDim(widthplot, heightplot)
159 160
160 161 # if self.created:
161 162 # self.__driver.closeFigure(self.fig)
162 163
163 164 if not self.created:
164 165 self.fig = self.__driver.createFigure(id=self.id,
165 166 wintitle=self.wintitle,
166 167 width=self.widthscreen,
167 168 height=self.heightscreen,
168 169 show=show)
169 170 else:
170 171 self.__driver.clearFigure(self.fig)
171 172
172 173 self.axesObjList = []
173 174 self.counter_imagwr = 0
174 175
175 176 self.created = True
176 177
177 178 def setDriver(self, driver=mpldriver):
178 179
179 180 self.__driver = driver
180 181
181 182 def setTitle(self, title):
182 183
183 184 self.__driver.setTitle(self.fig, title)
184 185
185 186 def setWinTitle(self, title):
186 187
187 188 self.__driver.setWinTitle(self.fig, title=title)
188 189
189 190 def setTextFromAxes(self, text):
190 191
191 192 raise NotImplementedError("This method has been replaced with Axes.setText")
192 193
193 194 def makeAxes(self, nrow, ncol, xpos, ypos, colspan, rowspan):
194 195
195 196 raise NotImplementedError("This method has been replaced with Axes.addAxes")
196 197
197 198 def addAxes(self, *args):
198 199 """
199 200
200 201 Input:
201 202 *args : Los parametros necesarios son
202 203 nrow, ncol, xpos, ypos, colspan, rowspan
203 204 """
204 205
205 206 axesObj = Axes(self.fig, *args)
206 207 self.axesObjList.append(axesObj)
207 208
208 209 def saveFigure(self, figpath, figfile, *args):
209 210
210 211 filename = os.path.join(figpath, figfile)
211 212
212 213 fullpath = os.path.split(filename)[0]
213 214
214 215 if not os.path.exists(fullpath):
215 216 subpath = os.path.split(fullpath)[0]
216 217
217 218 if not os.path.exists(subpath):
218 219 os.mkdir(subpath)
219 220
220 221 os.mkdir(fullpath)
221 222
222 223 self.__driver.saveFigure(self.fig, filename, *args)
223 224
224 225 def save(self, figpath, figfile=None, save=True, ftp=False, wr_period=1, thisDatetime=None, update_figfile=True):
225 226
226 227 self.counter_imagwr += 1
227 228 if self.counter_imagwr < wr_period:
228 229 return
229 230
230 231 self.counter_imagwr = 0
231 232
232 233 if save:
233 234
234 235 if not figfile:
235 236
236 237 if not thisDatetime:
237 238 raise ValueError("Saving figure: figfile or thisDatetime should be defined")
238 239 return
239 240
240 241 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
241 242 figfile = self.getFilename(name = str_datetime)
242 243
243 244 if self.figfile == None:
244 245 self.figfile = figfile
245 246
246 247 if update_figfile:
247 248 self.figfile = figfile
248 249
249 250 # store png plot to local folder
250 251 self.saveFigure(figpath, self.figfile)
251 252
252 253
253 254 if not ftp:
254 255 return
255 256
256 257 if not thisDatetime:
257 258 return
258 259
259 260 # store png plot to FTP server according to RT-Web format
260 261 ftp_filename = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
261 262 # ftp_filename = os.path.join(figpath, name)
262 263 self.saveFigure(figpath, ftp_filename)
263 264
264 265 def getNameToFtp(self, thisDatetime, FTP_WEI, EXP_CODE, SUB_EXP_CODE, PLOT_CODE, PLOT_POS):
265 266 YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
266 267 DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
267 268 FTP_WEI = '%2.2d'%FTP_WEI
268 269 EXP_CODE = '%3.3d'%EXP_CODE
269 270 SUB_EXP_CODE = '%2.2d'%SUB_EXP_CODE
270 271 PLOT_CODE = '%2.2d'%PLOT_CODE
271 272 PLOT_POS = '%2.2d'%PLOT_POS
272 273 name = YEAR_STR + DOY_STR + FTP_WEI + EXP_CODE + SUB_EXP_CODE + PLOT_CODE + PLOT_POS
273 274 return name
274 275
275 276 def draw(self):
276 277
277 278 self.__driver.draw(self.fig)
278 279
279 280 def run(self):
280 281
281 282 raise NotImplementedError
282 283
283 284 def close(self, show=False):
284 285
285 286 self.__driver.closeFigure(show=show, fig=self.fig)
286 287
287 288 axesList = property(getAxesObjList)
288 289
289 290
290 291 class Axes:
291 292
292 293 __driver = mpldriver
293 294 fig = None
294 295 ax = None
295 296 plot = None
296 297 __missing = 1E30
297 298 __firsttime = None
298 299
299 300 __showprofile = False
300 301
301 302 xmin = None
302 303 xmax = None
303 304 ymin = None
304 305 ymax = None
305 306 zmin = None
306 307 zmax = None
307 308
308 309 x_buffer = None
309 310 z_buffer = None
310 311
311 312 decimationx = None
312 313 decimationy = None
313 314
314 315 __MAXNUMX = 200
315 316 __MAXNUMY = 400
316 317
317 318 __MAXNUMTIME = 500
318 319
319 320 def __init__(self, *args):
320 321
321 322 """
322 323
323 324 Input:
324 325 *args : Los parametros necesarios son
325 326 fig, nrow, ncol, xpos, ypos, colspan, rowspan
326 327 """
327 328
328 329 ax = self.__driver.createAxes(*args)
329 330 self.fig = args[0]
330 331 self.ax = ax
331 332 self.plot = None
332 333
333 334 self.__firsttime = True
334 335 self.idlineList = []
335 336
336 337 self.x_buffer = numpy.array([])
337 338 self.z_buffer = numpy.array([])
338 339
339 340 def setText(self, text):
340 341
341 342 self.__driver.setAxesText(self.ax, text)
342 343
343 344 def setXAxisAsTime(self):
344 345 pass
345 346
346 347 def pline(self, x, y,
347 348 xmin=None, xmax=None,
348 349 ymin=None, ymax=None,
349 350 xlabel='', ylabel='',
350 351 title='',
351 352 **kwargs):
352 353
353 354 """
354 355
355 356 Input:
356 357 x :
357 358 y :
358 359 xmin :
359 360 xmax :
360 361 ymin :
361 362 ymax :
362 363 xlabel :
363 364 ylabel :
364 365 title :
365 366 **kwargs : Los parametros aceptados son
366 367
367 368 ticksize
368 369 ytick_visible
369 370 """
370 371
371 372 if self.__firsttime:
372 373
373 374 if xmin == None: xmin = numpy.nanmin(x)
374 375 if xmax == None: xmax = numpy.nanmax(x)
375 376 if ymin == None: ymin = numpy.nanmin(y)
376 377 if ymax == None: ymax = numpy.nanmax(y)
377 378
378 379 self.plot = self.__driver.createPline(self.ax, x, y,
379 380 xmin, xmax,
380 381 ymin, ymax,
381 382 xlabel=xlabel,
382 383 ylabel=ylabel,
383 384 title=title,
384 385 **kwargs)
385 386
386 387 self.idlineList.append(0)
387 388 self.__firsttime = False
388 389 return
389 390
390 391 self.__driver.pline(self.plot, x, y, xlabel=xlabel,
391 392 ylabel=ylabel,
392 393 title=title)
393 394
394 395 # self.__driver.pause()
395 396
396 397 def addpline(self, x, y, idline, **kwargs):
397 398 lines = self.ax.lines
398 399
399 400 if idline in self.idlineList:
400 401 self.__driver.set_linedata(self.ax, x, y, idline)
401 402
402 403 if idline not in(self.idlineList):
403 404 self.__driver.addpline(self.ax, x, y, **kwargs)
404 405 self.idlineList.append(idline)
405 406
406 407 return
407 408
408 409 def pmultiline(self, x, y,
409 410 xmin=None, xmax=None,
410 411 ymin=None, ymax=None,
411 412 xlabel='', ylabel='',
412 413 title='',
413 414 **kwargs):
414 415
415 416 if self.__firsttime:
416 417
417 418 if xmin == None: xmin = numpy.nanmin(x)
418 419 if xmax == None: xmax = numpy.nanmax(x)
419 420 if ymin == None: ymin = numpy.nanmin(y)
420 421 if ymax == None: ymax = numpy.nanmax(y)
421 422
422 423 self.plot = self.__driver.createPmultiline(self.ax, x, y,
423 424 xmin, xmax,
424 425 ymin, ymax,
425 426 xlabel=xlabel,
426 427 ylabel=ylabel,
427 428 title=title,
428 429 **kwargs)
429 430 self.__firsttime = False
430 431 return
431 432
432 433 self.__driver.pmultiline(self.plot, x, y, xlabel=xlabel,
433 434 ylabel=ylabel,
434 435 title=title)
435 436
436 437 # self.__driver.pause()
437 438
438 439 def pmultilineyaxis(self, x, y,
439 440 xmin=None, xmax=None,
440 441 ymin=None, ymax=None,
441 442 xlabel='', ylabel='',
442 443 title='',
443 444 **kwargs):
444 445
445 446 if self.__firsttime:
446 447
447 448 if xmin == None: xmin = numpy.nanmin(x)
448 449 if xmax == None: xmax = numpy.nanmax(x)
449 450 if ymin == None: ymin = numpy.nanmin(y)
450 451 if ymax == None: ymax = numpy.nanmax(y)
451 452
452 453 self.plot = self.__driver.createPmultilineYAxis(self.ax, x, y,
453 454 xmin, xmax,
454 455 ymin, ymax,
455 456 xlabel=xlabel,
456 457 ylabel=ylabel,
457 458 title=title,
458 459 **kwargs)
459 460 if self.xmin == None: self.xmin = xmin
460 461 if self.xmax == None: self.xmax = xmax
461 462 if self.ymin == None: self.ymin = ymin
462 463 if self.ymax == None: self.ymax = ymax
463 464
464 465 self.__firsttime = False
465 466 return
466 467
467 468 self.__driver.pmultilineyaxis(self.plot, x, y, xlabel=xlabel,
468 469 ylabel=ylabel,
469 470 title=title)
470 471
471 472 # self.__driver.pause()
472 473
473 474 def pcolor(self, x, y, z,
474 475 xmin=None, xmax=None,
475 476 ymin=None, ymax=None,
476 477 zmin=None, zmax=None,
477 478 xlabel='', ylabel='',
478 479 title='', colormap='jet',
479 480 **kwargs):
480 481
481 482 """
482 483 Input:
483 484 x :
484 485 y :
485 486 x :
486 487 xmin :
487 488 xmax :
488 489 ymin :
489 490 ymax :
490 491 zmin :
491 492 zmax :
492 493 xlabel :
493 494 ylabel :
494 495 title :
495 496 **kwargs : Los parametros aceptados son
496 497 ticksize=9,
497 498 cblabel=''
498 499 """
499 500
500 501 #Decimating data
501 502 xlen = len(x)
502 503 ylen = len(y)
503 504
504 505 decimationx = int(xlen/self.__MAXNUMX) + 1
505 506 decimationy = int(ylen/self.__MAXNUMY) + 1
506 507
507 508
508 509 x_buffer = x#[::decimationx]
509 510 y_buffer = y#[::decimationy]
510 511 z_buffer = z#[::decimationx, ::decimationy]
511 512 #===================================================
512 513
513 514 if self.__firsttime:
514 515
515 516 if xmin == None: xmin = numpy.nanmin(x)
516 517 if xmax == None: xmax = numpy.nanmax(x)
517 518 if ymin == None: ymin = numpy.nanmin(y)
518 519 if ymax == None: ymax = numpy.nanmax(y)
519 520 if zmin == None: zmin = numpy.nanmin(z)
520 521 if zmax == None: zmax = numpy.nanmax(z)
521 522
522 523
523 524 self.plot = self.__driver.createPcolor(self.ax, x_buffer,
524 525 y_buffer,
525 526 z_buffer,
526 527 xmin, xmax,
527 528 ymin, ymax,
528 529 zmin, zmax,
529 530 xlabel=xlabel,
530 531 ylabel=ylabel,
531 532 title=title,
532 533 colormap=colormap,
533 534 **kwargs)
534 535
535 536 if self.xmin == None: self.xmin = xmin
536 537 if self.xmax == None: self.xmax = xmax
537 538 if self.ymin == None: self.ymin = ymin
538 539 if self.ymax == None: self.ymax = ymax
539 540 if self.zmin == None: self.zmin = zmin
540 541 if self.zmax == None: self.zmax = zmax
541 542
542 543 self.__firsttime = False
543 544 return
544 545
545 546 self.__driver.pcolor(self.plot,
546 547 z_buffer,
547 548 xlabel=xlabel,
548 549 ylabel=ylabel,
549 550 title=title)
550 551
551 552 # self.__driver.pause()
552 553
553 554 def pcolorbuffer(self, x, y, z,
554 555 xmin=None, xmax=None,
555 556 ymin=None, ymax=None,
556 557 zmin=None, zmax=None,
557 558 xlabel='', ylabel='',
558 559 title='', rti = True, colormap='jet',
559 560 maxNumX = None, maxNumY = None,
560 561 **kwargs):
561 562
562 563 if maxNumX == None:
563 564 maxNumX = self.__MAXNUMTIME
564 565
565 566 if maxNumY == None:
566 567 maxNumY = self.__MAXNUMY
567 568
568 569 if self.__firsttime:
569 570 self.z_buffer = z
570 571 self.x_buffer = numpy.hstack((self.x_buffer, x))
571 572
572 573 if xmin == None: xmin = numpy.nanmin(x)
573 574 if xmax == None: xmax = numpy.nanmax(x)
574 575 if ymin == None: ymin = numpy.nanmin(y)
575 576 if ymax == None: ymax = numpy.nanmax(y)
576 577 if zmin == None: zmin = numpy.nanmin(z)
577 578 if zmax == None: zmax = numpy.nanmax(z)
578 579
579 580 self.plot = self.__driver.createPcolor(self.ax, self.x_buffer, y, z,
580 581 xmin, xmax,
581 582 ymin, ymax,
582 583 zmin, zmax,
583 584 xlabel=xlabel,
584 585 ylabel=ylabel,
585 586 title=title,
586 587 colormap=colormap,
587 588 **kwargs)
588 589
589 590 if self.xmin == None: self.xmin = xmin
590 591 if self.xmax == None: self.xmax = xmax
591 592 if self.ymin == None: self.ymin = ymin
592 593 if self.ymax == None: self.ymax = ymax
593 594 if self.zmin == None: self.zmin = zmin
594 595 if self.zmax == None: self.zmax = zmax
595 596
596 597 self.__firsttime = False
597 598 return
598 599
599 600 self.x_buffer = numpy.hstack((self.x_buffer[:-1], x[0], x[-1]))
600 601 self.z_buffer = numpy.hstack((self.z_buffer, z))
601 602 z_buffer = self.z_buffer.reshape(-1,len(y))
602 603
603 604 #Decimating data
604 605 xlen = len(self.x_buffer)
605 606 ylen = len(y)
606 607
607 608 decimationx = int(xlen/maxNumX) + 1
608 609 decimationy = int(ylen/maxNumY) + 1
609 610
610 611 x_buffer = self.x_buffer#[::decimationx]
611 612 y_buffer = y#[::decimationy]
612 613 z_buffer = z_buffer#[::decimationx, ::decimationy]
613 614 #===================================================
614 615
615 616 x_buffer, y_buffer, z_buffer = self.__fillGaps(x_buffer, y_buffer, z_buffer)
616 617
617 618 self.__driver.addpcolorbuffer(self.ax, x_buffer, y_buffer, z_buffer, self.zmin, self.zmax,
618 619 xlabel=xlabel,
619 620 ylabel=ylabel,
620 621 title=title,
621 622 colormap=colormap)
622 623
623 624 # self.__driver.pause()
624 625
625 626 def polar(self, x, y,
626 627 title='', xlabel='',ylabel='',**kwargs):
627 628
628 629 if self.__firsttime:
629 630 self.plot = self.__driver.createPolar(self.ax, x, y, title = title, xlabel = xlabel, ylabel = ylabel)
630 631 self.__firsttime = False
631 632 self.x_buffer = x
632 633 self.y_buffer = y
633 634 return
634 635
635 636 self.x_buffer = numpy.hstack((self.x_buffer,x))
636 637 self.y_buffer = numpy.hstack((self.y_buffer,y))
637 638 self.__driver.polar(self.plot, self.x_buffer, self.y_buffer, xlabel=xlabel,
638 639 ylabel=ylabel,
639 640 title=title)
640 641
641 642 # self.__driver.pause()
642 643
643 644 def __fillGaps(self, x_buffer, y_buffer, z_buffer):
644 645
645 646 if x_buffer.shape[0] < 2:
646 647 return x_buffer, y_buffer, z_buffer
647 648
648 649 deltas = x_buffer[1:] - x_buffer[0:-1]
649 650 x_median = numpy.median(deltas)
650 651
651 652 index = numpy.where(deltas > 5*x_median)
652 653
653 654 if len(index[0]) != 0:
654 655 z_buffer[index[0],::] = self.__missing
655 656 z_buffer = numpy.ma.masked_inside(z_buffer,0.99*self.__missing,1.01*self.__missing)
656 657
657 658 return x_buffer, y_buffer, z_buffer No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1542 +1,1587
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from .figure import Figure, isRealtime, isTimeInHourRange
11 11 from .plotting_codes import *
12 from schainpy.model.proc.jroproc_base import MPDecorator
12 13
14 from schainpy.utils import log
13 15
16 @MPDecorator
14 17 class SpectraPlot(Figure):
15 18
16 19 isConfig = None
17 20 __nsubplots = None
18 21
19 22 WIDTHPROF = None
20 23 HEIGHTPROF = None
21 24 PREFIX = 'spc'
22 25
23 def __init__(self, **kwargs):
24 Figure.__init__(self, **kwargs)
26 def __init__(self):#, **kwargs):
27 Figure.__init__(self)#, **kwargs)
25 28 self.isConfig = False
26 29 self.__nsubplots = 1
27
28 30 self.WIDTH = 250
29 31 self.HEIGHT = 250
30 32 self.WIDTHPROF = 120
31 33 self.HEIGHTPROF = 0
32 34 self.counter_imagwr = 0
33 35
34 36 self.PLOT_CODE = SPEC_CODE
35 37
36 38 self.FTP_WEI = None
37 39 self.EXP_CODE = None
38 40 self.SUB_EXP_CODE = None
39 41 self.PLOT_POS = None
40 42
41 43 self.__xfilter_ena = False
42 44 self.__yfilter_ena = False
43 45
44 46 def getSubplots(self):
45 47
46 48 ncol = int(numpy.sqrt(self.nplots)+0.9)
47 49 nrow = int(self.nplots*1./ncol + 0.9)
48 50
49 51 return nrow, ncol
50 52
51 53 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
52 54
53 55 self.__showprofile = showprofile
54 56 self.nplots = nplots
55 57
56 58 ncolspan = 1
57 59 colspan = 1
58 60 if showprofile:
59 61 ncolspan = 3
60 62 colspan = 2
61 63 self.__nsubplots = 2
62 64
63 65 self.createFigure(id = id,
64 66 wintitle = wintitle,
65 67 widthplot = self.WIDTH + self.WIDTHPROF,
66 68 heightplot = self.HEIGHT + self.HEIGHTPROF,
67 69 show=show)
68 70
69 71 nrow, ncol = self.getSubplots()
70 72
71 73 counter = 0
72 74 for y in range(nrow):
73 75 for x in range(ncol):
74 76
75 77 if counter >= self.nplots:
76 78 break
77 79
78 80 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
79 81
80 82 if showprofile:
81 83 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
82 84
83 85 counter += 1
84 86
85 87 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
86 88 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
87 89 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
88 90 server=None, folder=None, username=None, password=None,
89 91 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
90 92 xaxis="frequency", colormap='jet', normFactor=None):
91 93
92 94 """
93 95
94 96 Input:
95 97 dataOut :
96 98 id :
97 99 wintitle :
98 100 channelList :
99 101 showProfile :
100 102 xmin : None,
101 103 xmax : None,
102 104 ymin : None,
103 105 ymax : None,
104 106 zmin : None,
105 107 zmax : None
106 108 """
109 if dataOut.flagNoData:
110 return dataOut
111
107 112 if realtime:
108 113 if not(isRealtime(utcdatatime = dataOut.utctime)):
109 114 print('Skipping this plot function')
110 115 return
111 116
112 117 if channelList == None:
113 118 channelIndexList = dataOut.channelIndexList
114 119 else:
115 120 channelIndexList = []
116 121 for channel in channelList:
117 122 if channel not in dataOut.channelList:
118 123 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
119 124 channelIndexList.append(dataOut.channelList.index(channel))
120 125
121 126 if normFactor is None:
122 127 factor = dataOut.normFactor
123 128 else:
124 129 factor = normFactor
125 130 if xaxis == "frequency":
126 131 x = dataOut.getFreqRange(1)/1000.
127 132 xlabel = "Frequency (kHz)"
128 133
129 134 elif xaxis == "time":
130 135 x = dataOut.getAcfRange(1)
131 136 xlabel = "Time (ms)"
132 137
133 138 else:
134 139 x = dataOut.getVelRange(1)
135 140 xlabel = "Velocity (m/s)"
136 141
137 142 ylabel = "Range (Km)"
138 143
139 144 y = dataOut.getHeiRange()
140 145
141 146 z = dataOut.data_spc/factor
142 147 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
143 148 zdB = 10*numpy.log10(z)
144 149
145 150 avg = numpy.average(z, axis=1)
146 151 avgdB = 10*numpy.log10(avg)
147 152
148 153 noise = dataOut.getNoise()/factor
149 154 noisedB = 10*numpy.log10(noise)
150 155
151 156 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
152 157 title = wintitle + " Spectra"
153 158 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
154 159 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
155 160
156 161 if not self.isConfig:
157 162
158 163 nplots = len(channelIndexList)
159 164
160 165 self.setup(id=id,
161 166 nplots=nplots,
162 167 wintitle=wintitle,
163 168 showprofile=showprofile,
164 169 show=show)
165 170
166 171 if xmin == None: xmin = numpy.nanmin(x)
167 172 if xmax == None: xmax = numpy.nanmax(x)
168 173 if ymin == None: ymin = numpy.nanmin(y)
169 174 if ymax == None: ymax = numpy.nanmax(y)
170 175 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
171 176 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
172 177
173 178 self.FTP_WEI = ftp_wei
174 179 self.EXP_CODE = exp_code
175 180 self.SUB_EXP_CODE = sub_exp_code
176 181 self.PLOT_POS = plot_pos
177 182
178 183 self.isConfig = True
179 184
180 185 self.setWinTitle(title)
181 186
182 187 for i in range(self.nplots):
183 188 index = channelIndexList[i]
184 189 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
185 190 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
186 191 if len(dataOut.beam.codeList) != 0:
187 192 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
188 193
189 194 axes = self.axesList[i*self.__nsubplots]
190 195 axes.pcolor(x, y, zdB[index,:,:],
191 196 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
192 197 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
193 198 ticksize=9, cblabel='')
194 199
195 200 if self.__showprofile:
196 201 axes = self.axesList[i*self.__nsubplots +1]
197 202 axes.pline(avgdB[index,:], y,
198 203 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
199 204 xlabel='dB', ylabel='', title='',
200 205 ytick_visible=False,
201 206 grid='x')
202 207
203 208 noiseline = numpy.repeat(noisedB[index], len(y))
204 209 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
205 210
206 211 self.draw()
207 212
208 213 if figfile == None:
209 214 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
210 215 name = str_datetime
211 216 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
212 217 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
213 218 figfile = self.getFilename(name)
214 219
215 220 self.save(figpath=figpath,
216 221 figfile=figfile,
217 222 save=save,
218 223 ftp=ftp,
219 224 wr_period=wr_period,
220 225 thisDatetime=thisDatetime)
221 226
227 return dataOut
228 @MPDecorator
222 229 class CrossSpectraPlot(Figure):
223 230
224 231 isConfig = None
225 232 __nsubplots = None
226 233
227 234 WIDTH = None
228 235 HEIGHT = None
229 236 WIDTHPROF = None
230 237 HEIGHTPROF = None
231 238 PREFIX = 'cspc'
232 239
233 def __init__(self, **kwargs):
234 Figure.__init__(self, **kwargs)
240 def __init__(self):#, **kwargs):
241 Figure.__init__(self)#, **kwargs)
235 242 self.isConfig = False
236 243 self.__nsubplots = 4
237 244 self.counter_imagwr = 0
238 245 self.WIDTH = 250
239 246 self.HEIGHT = 250
240 247 self.WIDTHPROF = 0
241 248 self.HEIGHTPROF = 0
242 249
243 250 self.PLOT_CODE = CROSS_CODE
244 251 self.FTP_WEI = None
245 252 self.EXP_CODE = None
246 253 self.SUB_EXP_CODE = None
247 254 self.PLOT_POS = None
248 255
249 256 def getSubplots(self):
250 257
251 258 ncol = 4
252 259 nrow = self.nplots
253 260
254 261 return nrow, ncol
255 262
256 263 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
257 264
258 265 self.__showprofile = showprofile
259 266 self.nplots = nplots
260 267
261 268 ncolspan = 1
262 269 colspan = 1
263 270
264 271 self.createFigure(id = id,
265 272 wintitle = wintitle,
266 273 widthplot = self.WIDTH + self.WIDTHPROF,
267 274 heightplot = self.HEIGHT + self.HEIGHTPROF,
268 275 show=True)
269 276
270 277 nrow, ncol = self.getSubplots()
271 278
272 279 counter = 0
273 280 for y in range(nrow):
274 281 for x in range(ncol):
275 282 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
276 283
277 284 counter += 1
278 285
279 286 def run(self, dataOut, id, wintitle="", pairsList=None,
280 287 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
281 288 coh_min=None, coh_max=None, phase_min=None, phase_max=None,
282 289 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
283 290 power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
284 291 server=None, folder=None, username=None, password=None,
285 292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None,
286 293 xaxis='frequency'):
287 294
288 295 """
289 296
290 297 Input:
291 298 dataOut :
292 299 id :
293 300 wintitle :
294 301 channelList :
295 302 showProfile :
296 303 xmin : None,
297 304 xmax : None,
298 305 ymin : None,
299 306 ymax : None,
300 307 zmin : None,
301 308 zmax : None
302 309 """
303 310
311 if dataOut.flagNoData:
312 return dataOut
313
304 314 if pairsList == None:
305 315 pairsIndexList = dataOut.pairsIndexList
306 316 else:
307 317 pairsIndexList = []
308 318 for pair in pairsList:
309 319 if pair not in dataOut.pairsList:
310 320 raise ValueError("Pair %s is not in dataOut.pairsList" %str(pair))
311 321 pairsIndexList.append(dataOut.pairsList.index(pair))
312 322
313 323 if not pairsIndexList:
314 324 return
315 325
316 326 if len(pairsIndexList) > 4:
317 327 pairsIndexList = pairsIndexList[0:4]
318 328
319 329 if normFactor is None:
320 330 factor = dataOut.normFactor
321 331 else:
322 332 factor = normFactor
323 333 x = dataOut.getVelRange(1)
324 334 y = dataOut.getHeiRange()
325 335 z = dataOut.data_spc[:,:,:]/factor
326 336 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
327 337
328 338 noise = dataOut.noise/factor
329 339
330 340 zdB = 10*numpy.log10(z)
331 341 noisedB = 10*numpy.log10(noise)
332 342
333 343 if coh_min == None:
334 344 coh_min = 0.0
335 345 if coh_max == None:
336 346 coh_max = 1.0
337 347
338 348 if phase_min == None:
339 349 phase_min = -180
340 350 if phase_max == None:
341 351 phase_max = 180
342 352
343 353 #thisDatetime = dataOut.datatime
344 354 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
345 355 title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
346 356 # xlabel = "Velocity (m/s)"
347 357 ylabel = "Range (Km)"
348 358
349 359 if xaxis == "frequency":
350 360 x = dataOut.getFreqRange(1)/1000.
351 361 xlabel = "Frequency (kHz)"
352 362
353 363 elif xaxis == "time":
354 364 x = dataOut.getAcfRange(1)
355 365 xlabel = "Time (ms)"
356 366
357 367 else:
358 368 x = dataOut.getVelRange(1)
359 369 xlabel = "Velocity (m/s)"
360 370
361 371 if not self.isConfig:
362 372
363 373 nplots = len(pairsIndexList)
364 374
365 375 self.setup(id=id,
366 376 nplots=nplots,
367 377 wintitle=wintitle,
368 378 showprofile=False,
369 379 show=show)
370 380
371 381 avg = numpy.abs(numpy.average(z, axis=1))
372 382 avgdB = 10*numpy.log10(avg)
373 383
374 384 if xmin == None: xmin = numpy.nanmin(x)
375 385 if xmax == None: xmax = numpy.nanmax(x)
376 386 if ymin == None: ymin = numpy.nanmin(y)
377 387 if ymax == None: ymax = numpy.nanmax(y)
378 388 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
379 389 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
380 390
381 391 self.FTP_WEI = ftp_wei
382 392 self.EXP_CODE = exp_code
383 393 self.SUB_EXP_CODE = sub_exp_code
384 394 self.PLOT_POS = plot_pos
385 395
386 396 self.isConfig = True
387 397
388 398 self.setWinTitle(title)
389 399
390 400 for i in range(self.nplots):
391 401 pair = dataOut.pairsList[pairsIndexList[i]]
392 402
393 403 chan_index0 = dataOut.channelList.index(pair[0])
394 404 chan_index1 = dataOut.channelList.index(pair[1])
395 405
396 406 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
397 407 title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[chan_index0], str_datetime)
398 408 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index0,:,:]/factor)
399 409 axes0 = self.axesList[i*self.__nsubplots]
400 410 axes0.pcolor(x, y, zdB,
401 411 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
402 412 xlabel=xlabel, ylabel=ylabel, title=title,
403 413 ticksize=9, colormap=power_cmap, cblabel='')
404 414
405 415 title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[chan_index1], str_datetime)
406 416 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index1,:,:]/factor)
407 417 axes0 = self.axesList[i*self.__nsubplots+1]
408 418 axes0.pcolor(x, y, zdB,
409 419 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
410 420 xlabel=xlabel, ylabel=ylabel, title=title,
411 421 ticksize=9, colormap=power_cmap, cblabel='')
412 422
413 423 coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[chan_index0,:,:]*dataOut.data_spc[chan_index1,:,:])
414 424 coherence = numpy.abs(coherenceComplex)
415 425 # phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
416 426 phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
417 427
418 428 title = "Coherence Ch%d * Ch%d" %(pair[0], pair[1])
419 429 axes0 = self.axesList[i*self.__nsubplots+2]
420 430 axes0.pcolor(x, y, coherence,
421 431 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=coh_min, zmax=coh_max,
422 432 xlabel=xlabel, ylabel=ylabel, title=title,
423 433 ticksize=9, colormap=coherence_cmap, cblabel='')
424 434
425 435 title = "Phase Ch%d * Ch%d" %(pair[0], pair[1])
426 436 axes0 = self.axesList[i*self.__nsubplots+3]
427 437 axes0.pcolor(x, y, phase,
428 438 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
429 439 xlabel=xlabel, ylabel=ylabel, title=title,
430 440 ticksize=9, colormap=phase_cmap, cblabel='')
431 441
432 442
433 443
434 444 self.draw()
435 445
436 446 self.save(figpath=figpath,
437 447 figfile=figfile,
438 448 save=save,
439 449 ftp=ftp,
440 450 wr_period=wr_period,
441 451 thisDatetime=thisDatetime)
442 452
453 return dataOut
443 454
455 @MPDecorator
444 456 class RTIPlot(Figure):
445 457
446 458 __isConfig = None
447 459 __nsubplots = None
448 460
449 461 WIDTHPROF = None
450 462 HEIGHTPROF = None
451 463 PREFIX = 'rti'
452 464
453 def __init__(self, **kwargs):
465 def __init__(self):#, **kwargs):
454 466
455 Figure.__init__(self, **kwargs)
467 Figure.__init__(self)#, **kwargs)
456 468 self.timerange = None
457 469 self.isConfig = False
458 470 self.__nsubplots = 1
459 471
460 472 self.WIDTH = 800
461 473 self.HEIGHT = 180
462 474 self.WIDTHPROF = 120
463 475 self.HEIGHTPROF = 0
464 476 self.counter_imagwr = 0
465 477
466 478 self.PLOT_CODE = RTI_CODE
467 479
468 480 self.FTP_WEI = None
469 481 self.EXP_CODE = None
470 482 self.SUB_EXP_CODE = None
471 483 self.PLOT_POS = None
472 484 self.tmin = None
473 485 self.tmax = None
474 486
475 487 self.xmin = None
476 488 self.xmax = None
477 489
478 490 self.figfile = None
479 491
480 492 def getSubplots(self):
481 493
482 494 ncol = 1
483 495 nrow = self.nplots
484 496
485 497 return nrow, ncol
486 498
487 499 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
488 500
489 501 self.__showprofile = showprofile
490 502 self.nplots = nplots
491 503
492 504 ncolspan = 1
493 505 colspan = 1
494 506 if showprofile:
495 507 ncolspan = 7
496 508 colspan = 6
497 509 self.__nsubplots = 2
498 510
499 511 self.createFigure(id = id,
500 512 wintitle = wintitle,
501 513 widthplot = self.WIDTH + self.WIDTHPROF,
502 514 heightplot = self.HEIGHT + self.HEIGHTPROF,
503 515 show=show)
504 516
505 517 nrow, ncol = self.getSubplots()
506 518
507 519 counter = 0
508 520 for y in range(nrow):
509 521 for x in range(ncol):
510 522
511 523 if counter >= self.nplots:
512 524 break
513 525
514 526 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
515 527
516 528 if showprofile:
517 529 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
518 530
519 531 counter += 1
520 532
521 533 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
522 534 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
523 535 timerange=None, colormap='jet',
524 536 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
525 537 server=None, folder=None, username=None, password=None,
526 538 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None):
527 539
528 540 """
529 541
530 542 Input:
531 543 dataOut :
532 544 id :
533 545 wintitle :
534 546 channelList :
535 547 showProfile :
536 548 xmin : None,
537 549 xmax : None,
538 550 ymin : None,
539 551 ymax : None,
540 552 zmin : None,
541 553 zmax : None
542 554 """
555 if dataOut.flagNoData:
556 return dataOut
543 557
544 558 #colormap = kwargs.get('colormap', 'jet')
545 559 if HEIGHT is not None:
546 560 self.HEIGHT = HEIGHT
547 561
548 562 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
549 563 return
550 564
551 565 if channelList == None:
552 566 channelIndexList = dataOut.channelIndexList
553 567 else:
554 568 channelIndexList = []
555 569 for channel in channelList:
556 570 if channel not in dataOut.channelList:
557 571 raise ValueError("Channel %d is not in dataOut.channelList")
558 572 channelIndexList.append(dataOut.channelList.index(channel))
559 573
560 574 if normFactor is None:
561 575 factor = dataOut.normFactor
562 576 else:
563 577 factor = normFactor
564 578
565 579 # factor = dataOut.normFactor
566 580 x = dataOut.getTimeRange()
567 581 y = dataOut.getHeiRange()
568 582
569 583 z = dataOut.data_spc/factor
570 584 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
571 585 avg = numpy.average(z, axis=1)
572 586 avgdB = 10.*numpy.log10(avg)
573 587 # avgdB = dataOut.getPower()
574 588
575 589
576 590 thisDatetime = dataOut.datatime
577 591 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
578 592 title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
579 593 xlabel = ""
580 594 ylabel = "Range (Km)"
581 595
582 596 update_figfile = False
583 597
584 598 if self.xmax is not None and dataOut.ltctime >= self.xmax: #yong
585 599 self.counter_imagwr = wr_period
586 600 self.isConfig = False
587 601 update_figfile = True
588 602
589 603 if not self.isConfig:
590 604
591 605 nplots = len(channelIndexList)
592 606
593 607 self.setup(id=id,
594 608 nplots=nplots,
595 609 wintitle=wintitle,
596 610 showprofile=showprofile,
597 611 show=show)
598 612
599 613 if timerange != None:
600 614 self.timerange = timerange
601 615
602 616 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
603 617
604 618 noise = dataOut.noise/factor
605 619 noisedB = 10*numpy.log10(noise)
606 620
607 621 if ymin == None: ymin = numpy.nanmin(y)
608 622 if ymax == None: ymax = numpy.nanmax(y)
609 623 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
610 624 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
611 625
612 626 self.FTP_WEI = ftp_wei
613 627 self.EXP_CODE = exp_code
614 628 self.SUB_EXP_CODE = sub_exp_code
615 629 self.PLOT_POS = plot_pos
616 630
617 631 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
618 632 self.isConfig = True
619 633 self.figfile = figfile
620 634 update_figfile = True
621 635
622 636 self.setWinTitle(title)
623 637
624 638 for i in range(self.nplots):
625 639 index = channelIndexList[i]
626 640 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
627 641 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
628 642 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
629 643 axes = self.axesList[i*self.__nsubplots]
630 644 zdB = avgdB[index].reshape((1,-1))
631 645 axes.pcolorbuffer(x, y, zdB,
632 646 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
633 647 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
634 648 ticksize=9, cblabel='', cbsize="1%", colormap=colormap)
635 649
636 650 if self.__showprofile:
637 651 axes = self.axesList[i*self.__nsubplots +1]
638 652 axes.pline(avgdB[index], y,
639 653 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
640 654 xlabel='dB', ylabel='', title='',
641 655 ytick_visible=False,
642 656 grid='x')
643 657
644 658 self.draw()
645 659
646 660 self.save(figpath=figpath,
647 661 figfile=figfile,
648 662 save=save,
649 663 ftp=ftp,
650 664 wr_period=wr_period,
651 665 thisDatetime=thisDatetime,
652 666 update_figfile=update_figfile)
667 return dataOut
653 668
669 @MPDecorator
654 670 class CoherenceMap(Figure):
655 671 isConfig = None
656 672 __nsubplots = None
657 673
658 674 WIDTHPROF = None
659 675 HEIGHTPROF = None
660 676 PREFIX = 'cmap'
661 677
662 def __init__(self, **kwargs):
663 Figure.__init__(self, **kwargs)
678 def __init__(self):#, **kwargs):
679 Figure.__init__(self)#, **kwargs)
664 680 self.timerange = 2*60*60
665 681 self.isConfig = False
666 682 self.__nsubplots = 1
667 683
668 684 self.WIDTH = 800
669 685 self.HEIGHT = 180
670 686 self.WIDTHPROF = 120
671 687 self.HEIGHTPROF = 0
672 688 self.counter_imagwr = 0
673 689
674 690 self.PLOT_CODE = COH_CODE
675 691
676 692 self.FTP_WEI = None
677 693 self.EXP_CODE = None
678 694 self.SUB_EXP_CODE = None
679 695 self.PLOT_POS = None
680 696 self.counter_imagwr = 0
681 697
682 698 self.xmin = None
683 699 self.xmax = None
684 700
685 701 def getSubplots(self):
686 702 ncol = 1
687 703 nrow = self.nplots*2
688 704
689 705 return nrow, ncol
690 706
691 707 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
692 708 self.__showprofile = showprofile
693 709 self.nplots = nplots
694 710
695 711 ncolspan = 1
696 712 colspan = 1
697 713 if showprofile:
698 714 ncolspan = 7
699 715 colspan = 6
700 716 self.__nsubplots = 2
701 717
702 718 self.createFigure(id = id,
703 719 wintitle = wintitle,
704 720 widthplot = self.WIDTH + self.WIDTHPROF,
705 721 heightplot = self.HEIGHT + self.HEIGHTPROF,
706 722 show=True)
707 723
708 724 nrow, ncol = self.getSubplots()
709 725
710 726 for y in range(nrow):
711 727 for x in range(ncol):
712 728
713 729 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
714 730
715 731 if showprofile:
716 732 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
717 733
718 734 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
719 735 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
720 736 timerange=None, phase_min=None, phase_max=None,
721 737 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
722 738 coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
723 739 server=None, folder=None, username=None, password=None,
724 740 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
725 741
742
743 if dataOut.flagNoData:
744 return dataOut
745
726 746 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
727 747 return
728 748
729 749 if pairsList == None:
730 750 pairsIndexList = dataOut.pairsIndexList
731 751 else:
732 752 pairsIndexList = []
733 753 for pair in pairsList:
734 754 if pair not in dataOut.pairsList:
735 755 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
736 756 pairsIndexList.append(dataOut.pairsList.index(pair))
737 757
738 758 if pairsIndexList == []:
739 759 return
740 760
741 761 if len(pairsIndexList) > 4:
742 762 pairsIndexList = pairsIndexList[0:4]
743 763
744 764 if phase_min == None:
745 765 phase_min = -180
746 766 if phase_max == None:
747 767 phase_max = 180
748 768
749 769 x = dataOut.getTimeRange()
750 770 y = dataOut.getHeiRange()
751 771
752 772 thisDatetime = dataOut.datatime
753 773
754 774 title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
755 775 xlabel = ""
756 776 ylabel = "Range (Km)"
757 777 update_figfile = False
758 778
759 779 if not self.isConfig:
760 780 nplots = len(pairsIndexList)
761 781 self.setup(id=id,
762 782 nplots=nplots,
763 783 wintitle=wintitle,
764 784 showprofile=showprofile,
765 785 show=show)
766 786
767 787 if timerange != None:
768 788 self.timerange = timerange
769 789
770 790 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
771 791
772 792 if ymin == None: ymin = numpy.nanmin(y)
773 793 if ymax == None: ymax = numpy.nanmax(y)
774 794 if zmin == None: zmin = 0.
775 795 if zmax == None: zmax = 1.
776 796
777 797 self.FTP_WEI = ftp_wei
778 798 self.EXP_CODE = exp_code
779 799 self.SUB_EXP_CODE = sub_exp_code
780 800 self.PLOT_POS = plot_pos
781 801
782 802 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
783 803
784 804 self.isConfig = True
785 805 update_figfile = True
786 806
787 807 self.setWinTitle(title)
788 808
789 809 for i in range(self.nplots):
790 810
791 811 pair = dataOut.pairsList[pairsIndexList[i]]
792 812
793 813 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
794 814 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
795 815 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
796 816
797 817
798 818 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
799 819 coherence = numpy.abs(avgcoherenceComplex)
800 820
801 821 z = coherence.reshape((1,-1))
802 822
803 823 counter = 0
804 824
805 825 title = "Coherence Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
806 826 axes = self.axesList[i*self.__nsubplots*2]
807 827 axes.pcolorbuffer(x, y, z,
808 828 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
809 829 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
810 830 ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
811 831
812 832 if self.__showprofile:
813 833 counter += 1
814 834 axes = self.axesList[i*self.__nsubplots*2 + counter]
815 835 axes.pline(coherence, y,
816 836 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
817 837 xlabel='', ylabel='', title='', ticksize=7,
818 838 ytick_visible=False, nxticks=5,
819 839 grid='x')
820 840
821 841 counter += 1
822 842
823 843 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
824 844
825 845 z = phase.reshape((1,-1))
826 846
827 847 title = "Phase Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
828 848 axes = self.axesList[i*self.__nsubplots*2 + counter]
829 849 axes.pcolorbuffer(x, y, z,
830 850 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
831 851 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
832 852 ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
833 853
834 854 if self.__showprofile:
835 855 counter += 1
836 856 axes = self.axesList[i*self.__nsubplots*2 + counter]
837 857 axes.pline(phase, y,
838 858 xmin=phase_min, xmax=phase_max, ymin=ymin, ymax=ymax,
839 859 xlabel='', ylabel='', title='', ticksize=7,
840 860 ytick_visible=False, nxticks=4,
841 861 grid='x')
842 862
843 863 self.draw()
844 864
845 865 if dataOut.ltctime >= self.xmax:
846 866 self.counter_imagwr = wr_period
847 867 self.isConfig = False
848 868 update_figfile = True
849 869
850 870 self.save(figpath=figpath,
851 871 figfile=figfile,
852 872 save=save,
853 873 ftp=ftp,
854 874 wr_period=wr_period,
855 875 thisDatetime=thisDatetime,
856 876 update_figfile=update_figfile)
857 877
878 return dataOut
879
880 @MPDecorator
858 881 class PowerProfilePlot(Figure):
859 882
860 883 isConfig = None
861 884 __nsubplots = None
862 885
863 886 WIDTHPROF = None
864 887 HEIGHTPROF = None
865 888 PREFIX = 'spcprofile'
866 889
867 def __init__(self, **kwargs):
868 Figure.__init__(self, **kwargs)
890 def __init__(self):#, **kwargs):
891 Figure.__init__(self)#, **kwargs)
869 892 self.isConfig = False
870 893 self.__nsubplots = 1
871 894
872 895 self.PLOT_CODE = POWER_CODE
873 896
874 897 self.WIDTH = 300
875 898 self.HEIGHT = 500
876 899 self.counter_imagwr = 0
877 900
878 901 def getSubplots(self):
879 902 ncol = 1
880 903 nrow = 1
881 904
882 905 return nrow, ncol
883 906
884 907 def setup(self, id, nplots, wintitle, show):
885 908
886 909 self.nplots = nplots
887 910
888 911 ncolspan = 1
889 912 colspan = 1
890 913
891 914 self.createFigure(id = id,
892 915 wintitle = wintitle,
893 916 widthplot = self.WIDTH,
894 917 heightplot = self.HEIGHT,
895 918 show=show)
896 919
897 920 nrow, ncol = self.getSubplots()
898 921
899 922 counter = 0
900 923 for y in range(nrow):
901 924 for x in range(ncol):
902 925 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
903 926
904 927 def run(self, dataOut, id, wintitle="", channelList=None,
905 928 xmin=None, xmax=None, ymin=None, ymax=None,
906 929 save=False, figpath='./', figfile=None, show=True,
907 930 ftp=False, wr_period=1, server=None,
908 931 folder=None, username=None, password=None):
909 932
933 if dataOut.flagNoData:
934 return dataOut
935
910 936
911 937 if channelList == None:
912 938 channelIndexList = dataOut.channelIndexList
913 939 channelList = dataOut.channelList
914 940 else:
915 941 channelIndexList = []
916 942 for channel in channelList:
917 943 if channel not in dataOut.channelList:
918 944 raise ValueError("Channel %d is not in dataOut.channelList")
919 945 channelIndexList.append(dataOut.channelList.index(channel))
920 946
921 947 factor = dataOut.normFactor
922 948
923 949 y = dataOut.getHeiRange()
924 950
925 951 #for voltage
926 952 if dataOut.type == 'Voltage':
927 953 x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
928 954 x = x.real
929 955 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
930 956
931 957 #for spectra
932 958 if dataOut.type == 'Spectra':
933 959 x = dataOut.data_spc[channelIndexList,:,:]/factor
934 960 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
935 961 x = numpy.average(x, axis=1)
936 962
937 963
938 964 xdB = 10*numpy.log10(x)
939 965
940 966 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
941 967 title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
942 968 xlabel = "dB"
943 969 ylabel = "Range (Km)"
944 970
945 971 if not self.isConfig:
946 972
947 973 nplots = 1
948 974
949 975 self.setup(id=id,
950 976 nplots=nplots,
951 977 wintitle=wintitle,
952 978 show=show)
953 979
954 980 if ymin == None: ymin = numpy.nanmin(y)
955 981 if ymax == None: ymax = numpy.nanmax(y)
956 982 if xmin == None: xmin = numpy.nanmin(xdB)*0.9
957 983 if xmax == None: xmax = numpy.nanmax(xdB)*1.1
958 984
959 985 self.isConfig = True
960 986
961 987 self.setWinTitle(title)
962 988
963 989 title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
964 990 axes = self.axesList[0]
965 991
966 992 legendlabels = ["channel %d"%x for x in channelList]
967 993 axes.pmultiline(xdB, y,
968 994 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
969 995 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
970 996 ytick_visible=True, nxticks=5,
971 997 grid='x')
972 998
973 999 self.draw()
974 1000
975 1001 self.save(figpath=figpath,
976 1002 figfile=figfile,
977 1003 save=save,
978 1004 ftp=ftp,
979 1005 wr_period=wr_period,
980 1006 thisDatetime=thisDatetime)
1007
1008 return dataOut
981 1009
1010 @MPDecorator
982 1011 class SpectraCutPlot(Figure):
983 1012
984 1013 isConfig = None
985 1014 __nsubplots = None
986 1015
987 1016 WIDTHPROF = None
988 1017 HEIGHTPROF = None
989 1018 PREFIX = 'spc_cut'
990 1019
991 def __init__(self, **kwargs):
992 Figure.__init__(self, **kwargs)
1020 def __init__(self):#, **kwargs):
1021 Figure.__init__(self)#, **kwargs)
993 1022 self.isConfig = False
994 1023 self.__nsubplots = 1
995 1024
996 1025 self.PLOT_CODE = POWER_CODE
997 1026
998 1027 self.WIDTH = 700
999 1028 self.HEIGHT = 500
1000 1029 self.counter_imagwr = 0
1001 1030
1002 1031 def getSubplots(self):
1003 1032 ncol = 1
1004 1033 nrow = 1
1005 1034
1006 1035 return nrow, ncol
1007 1036
1008 1037 def setup(self, id, nplots, wintitle, show):
1009 1038
1010 1039 self.nplots = nplots
1011 1040
1012 1041 ncolspan = 1
1013 1042 colspan = 1
1014 1043
1015 1044 self.createFigure(id = id,
1016 1045 wintitle = wintitle,
1017 1046 widthplot = self.WIDTH,
1018 1047 heightplot = self.HEIGHT,
1019 1048 show=show)
1020 1049
1021 1050 nrow, ncol = self.getSubplots()
1022 1051
1023 1052 counter = 0
1024 1053 for y in range(nrow):
1025 1054 for x in range(ncol):
1026 1055 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1027 1056
1028 1057 def run(self, dataOut, id, wintitle="", channelList=None,
1029 1058 xmin=None, xmax=None, ymin=None, ymax=None,
1030 1059 save=False, figpath='./', figfile=None, show=True,
1031 1060 ftp=False, wr_period=1, server=None,
1032 1061 folder=None, username=None, password=None,
1033 1062 xaxis="frequency"):
1034 1063
1064 if dataOut.flagNoData:
1065 return dataOut
1035 1066
1036 1067 if channelList == None:
1037 1068 channelIndexList = dataOut.channelIndexList
1038 1069 channelList = dataOut.channelList
1039 1070 else:
1040 1071 channelIndexList = []
1041 1072 for channel in channelList:
1042 1073 if channel not in dataOut.channelList:
1043 1074 raise ValueError("Channel %d is not in dataOut.channelList")
1044 1075 channelIndexList.append(dataOut.channelList.index(channel))
1045 1076
1046 1077 factor = dataOut.normFactor
1047 1078
1048 1079 y = dataOut.getHeiRange()
1049 1080
1050 1081 z = dataOut.data_spc/factor
1051 1082 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1052 1083
1053 1084 hei_index = numpy.arange(25)*3 + 20
1054 1085
1055 1086 if xaxis == "frequency":
1056 1087 x = dataOut.getFreqRange()/1000.
1057 1088 zdB = 10*numpy.log10(z[0,:,hei_index])
1058 1089 xlabel = "Frequency (kHz)"
1059 1090 ylabel = "Power (dB)"
1060 1091
1061 1092 elif xaxis == "time":
1062 1093 x = dataOut.getAcfRange()
1063 1094 zdB = z[0,:,hei_index]
1064 1095 xlabel = "Time (ms)"
1065 1096 ylabel = "ACF"
1066 1097
1067 1098 else:
1068 1099 x = dataOut.getVelRange()
1069 1100 zdB = 10*numpy.log10(z[0,:,hei_index])
1070 1101 xlabel = "Velocity (m/s)"
1071 1102 ylabel = "Power (dB)"
1072 1103
1073 1104 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1074 1105 title = wintitle + " Range Cuts %s" %(thisDatetime.strftime("%d-%b-%Y"))
1075 1106
1076 1107 if not self.isConfig:
1077 1108
1078 1109 nplots = 1
1079 1110
1080 1111 self.setup(id=id,
1081 1112 nplots=nplots,
1082 1113 wintitle=wintitle,
1083 1114 show=show)
1084 1115
1085 1116 if xmin == None: xmin = numpy.nanmin(x)*0.9
1086 1117 if xmax == None: xmax = numpy.nanmax(x)*1.1
1087 1118 if ymin == None: ymin = numpy.nanmin(zdB)
1088 1119 if ymax == None: ymax = numpy.nanmax(zdB)
1089 1120
1090 1121 self.isConfig = True
1091 1122
1092 1123 self.setWinTitle(title)
1093 1124
1094 1125 title = "Spectra Cuts: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1095 1126 axes = self.axesList[0]
1096 1127
1097 1128 legendlabels = ["Range = %dKm" %y[i] for i in hei_index]
1098 1129
1099 1130 axes.pmultilineyaxis( x, zdB,
1100 1131 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1101 1132 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
1102 1133 ytick_visible=True, nxticks=5,
1103 1134 grid='x')
1104 1135
1105 1136 self.draw()
1106 1137
1107 1138 self.save(figpath=figpath,
1108 1139 figfile=figfile,
1109 1140 save=save,
1110 1141 ftp=ftp,
1111 1142 wr_period=wr_period,
1112 1143 thisDatetime=thisDatetime)
1113 1144
1145 return dataOut
1146
1147 @MPDecorator
1114 1148 class Noise(Figure):
1115 1149
1116 1150 isConfig = None
1117 1151 __nsubplots = None
1118 1152
1119 1153 PREFIX = 'noise'
1120 1154
1121 1155
1122 def __init__(self, **kwargs):
1123 Figure.__init__(self, **kwargs)
1156 def __init__(self):#, **kwargs):
1157 Figure.__init__(self)#, **kwargs)
1124 1158 self.timerange = 24*60*60
1125 1159 self.isConfig = False
1126 1160 self.__nsubplots = 1
1127 1161 self.counter_imagwr = 0
1128 1162 self.WIDTH = 800
1129 1163 self.HEIGHT = 400
1130 1164 self.WIDTHPROF = 120
1131 1165 self.HEIGHTPROF = 0
1132 1166 self.xdata = None
1133 1167 self.ydata = None
1134 1168
1135 1169 self.PLOT_CODE = NOISE_CODE
1136 1170
1137 1171 self.FTP_WEI = None
1138 1172 self.EXP_CODE = None
1139 1173 self.SUB_EXP_CODE = None
1140 1174 self.PLOT_POS = None
1141 1175 self.figfile = None
1142 1176
1143 1177 self.xmin = None
1144 1178 self.xmax = None
1145 1179
1146 1180 def getSubplots(self):
1147 1181
1148 1182 ncol = 1
1149 1183 nrow = 1
1150 1184
1151 1185 return nrow, ncol
1152 1186
1153 1187 def openfile(self, filename):
1154 1188 dirname = os.path.dirname(filename)
1155 1189
1156 1190 if not os.path.exists(dirname):
1157 1191 os.mkdir(dirname)
1158 1192
1159 1193 f = open(filename,'w+')
1160 1194 f.write('\n\n')
1161 1195 f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
1162 1196 f.write('DD MM YYYY HH MM SS Channel0 Channel1 Channel2 Channel3\n\n' )
1163 1197 f.close()
1164 1198
1165 1199 def save_data(self, filename_phase, data, data_datetime):
1166 1200
1167 1201 f=open(filename_phase,'a')
1168 1202
1169 1203 timetuple_data = data_datetime.timetuple()
1170 1204 day = str(timetuple_data.tm_mday)
1171 1205 month = str(timetuple_data.tm_mon)
1172 1206 year = str(timetuple_data.tm_year)
1173 1207 hour = str(timetuple_data.tm_hour)
1174 1208 minute = str(timetuple_data.tm_min)
1175 1209 second = str(timetuple_data.tm_sec)
1176 1210
1177 1211 data_msg = ''
1178 1212 for i in range(len(data)):
1179 1213 data_msg += str(data[i]) + ' '
1180 1214
1181 1215 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' ' + data_msg + '\n')
1182 1216 f.close()
1183 1217
1184 1218
1185 1219 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1186 1220
1187 1221 self.__showprofile = showprofile
1188 1222 self.nplots = nplots
1189 1223
1190 1224 ncolspan = 7
1191 1225 colspan = 6
1192 1226 self.__nsubplots = 2
1193 1227
1194 1228 self.createFigure(id = id,
1195 1229 wintitle = wintitle,
1196 1230 widthplot = self.WIDTH+self.WIDTHPROF,
1197 1231 heightplot = self.HEIGHT+self.HEIGHTPROF,
1198 1232 show=show)
1199 1233
1200 1234 nrow, ncol = self.getSubplots()
1201 1235
1202 1236 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1203 1237
1204 1238
1205 1239 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
1206 1240 xmin=None, xmax=None, ymin=None, ymax=None,
1207 1241 timerange=None,
1208 1242 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1209 1243 server=None, folder=None, username=None, password=None,
1210 1244 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1211 1245
1246 if dataOut.flagNoData:
1247 return dataOut
1248
1212 1249 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1213 1250 return
1214 1251
1215 1252 if channelList == None:
1216 1253 channelIndexList = dataOut.channelIndexList
1217 1254 channelList = dataOut.channelList
1218 1255 else:
1219 1256 channelIndexList = []
1220 1257 for channel in channelList:
1221 1258 if channel not in dataOut.channelList:
1222 1259 raise ValueError("Channel %d is not in dataOut.channelList")
1223 1260 channelIndexList.append(dataOut.channelList.index(channel))
1224 1261
1225 1262 x = dataOut.getTimeRange()
1226 1263 #y = dataOut.getHeiRange()
1227 1264 factor = dataOut.normFactor
1228 1265 noise = dataOut.noise[channelIndexList]/factor
1229 1266 noisedB = 10*numpy.log10(noise)
1230 1267
1231 1268 thisDatetime = dataOut.datatime
1232 1269
1233 1270 title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1234 1271 xlabel = ""
1235 1272 ylabel = "Intensity (dB)"
1236 1273 update_figfile = False
1237 1274
1238 1275 if not self.isConfig:
1239 1276
1240 1277 nplots = 1
1241 1278
1242 1279 self.setup(id=id,
1243 1280 nplots=nplots,
1244 1281 wintitle=wintitle,
1245 1282 showprofile=showprofile,
1246 1283 show=show)
1247 1284
1248 1285 if timerange != None:
1249 1286 self.timerange = timerange
1250 1287
1251 1288 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1252 1289
1253 1290 if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
1254 1291 if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
1255 1292
1256 1293 self.FTP_WEI = ftp_wei
1257 1294 self.EXP_CODE = exp_code
1258 1295 self.SUB_EXP_CODE = sub_exp_code
1259 1296 self.PLOT_POS = plot_pos
1260 1297
1261 1298
1262 1299 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1263 1300 self.isConfig = True
1264 1301 self.figfile = figfile
1265 1302 self.xdata = numpy.array([])
1266 1303 self.ydata = numpy.array([])
1267 1304
1268 1305 update_figfile = True
1269 1306
1270 1307 #open file beacon phase
1271 1308 path = '%s%03d' %(self.PREFIX, self.id)
1272 1309 noise_file = os.path.join(path,'%s.txt'%self.name)
1273 1310 self.filename_noise = os.path.join(figpath,noise_file)
1274 1311
1275 1312 self.setWinTitle(title)
1276 1313
1277 1314 title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1278 1315
1279 1316 legendlabels = ["channel %d"%(idchannel) for idchannel in channelList]
1280 1317 axes = self.axesList[0]
1281 1318
1282 1319 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1283 1320
1284 1321 if len(self.ydata)==0:
1285 1322 self.ydata = noisedB.reshape(-1,1)
1286 1323 else:
1287 1324 self.ydata = numpy.hstack((self.ydata, noisedB.reshape(-1,1)))
1288 1325
1289 1326
1290 1327 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1291 1328 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1292 1329 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1293 1330 XAxisAsTime=True, grid='both'
1294 1331 )
1295 1332
1296 1333 self.draw()
1297 1334
1298 1335 if dataOut.ltctime >= self.xmax:
1299 1336 self.counter_imagwr = wr_period
1300 1337 self.isConfig = False
1301 1338 update_figfile = True
1302 1339
1303 1340 self.save(figpath=figpath,
1304 1341 figfile=figfile,
1305 1342 save=save,
1306 1343 ftp=ftp,
1307 1344 wr_period=wr_period,
1308 1345 thisDatetime=thisDatetime,
1309 1346 update_figfile=update_figfile)
1310 1347
1311 1348 #store data beacon phase
1312 1349 if save:
1313 1350 self.save_data(self.filename_noise, noisedB, thisDatetime)
1314 1351
1352 return dataOut
1353
1354 @MPDecorator
1315 1355 class BeaconPhase(Figure):
1316 1356
1317 1357 __isConfig = None
1318 1358 __nsubplots = None
1319 1359
1320 1360 PREFIX = 'beacon_phase'
1321 1361
1322 def __init__(self, **kwargs):
1323 Figure.__init__(self, **kwargs)
1362 def __init__(self):#, **kwargs):
1363 Figure.__init__(self)#, **kwargs)
1324 1364 self.timerange = 24*60*60
1325 1365 self.isConfig = False
1326 1366 self.__nsubplots = 1
1327 1367 self.counter_imagwr = 0
1328 1368 self.WIDTH = 800
1329 1369 self.HEIGHT = 400
1330 1370 self.WIDTHPROF = 120
1331 1371 self.HEIGHTPROF = 0
1332 1372 self.xdata = None
1333 1373 self.ydata = None
1334 1374
1335 1375 self.PLOT_CODE = BEACON_CODE
1336 1376
1337 1377 self.FTP_WEI = None
1338 1378 self.EXP_CODE = None
1339 1379 self.SUB_EXP_CODE = None
1340 1380 self.PLOT_POS = None
1341 1381
1342 1382 self.filename_phase = None
1343 1383
1344 1384 self.figfile = None
1345 1385
1346 1386 self.xmin = None
1347 1387 self.xmax = None
1348 1388
1349 1389 def getSubplots(self):
1350 1390
1351 1391 ncol = 1
1352 1392 nrow = 1
1353 1393
1354 1394 return nrow, ncol
1355 1395
1356 1396 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1357 1397
1358 1398 self.__showprofile = showprofile
1359 1399 self.nplots = nplots
1360 1400
1361 1401 ncolspan = 7
1362 1402 colspan = 6
1363 1403 self.__nsubplots = 2
1364 1404
1365 1405 self.createFigure(id = id,
1366 1406 wintitle = wintitle,
1367 1407 widthplot = self.WIDTH+self.WIDTHPROF,
1368 1408 heightplot = self.HEIGHT+self.HEIGHTPROF,
1369 1409 show=show)
1370 1410
1371 1411 nrow, ncol = self.getSubplots()
1372 1412
1373 1413 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1374 1414
1375 1415 def save_phase(self, filename_phase):
1376 1416 f = open(filename_phase,'w+')
1377 1417 f.write('\n\n')
1378 1418 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
1379 1419 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
1380 1420 f.close()
1381 1421
1382 1422 def save_data(self, filename_phase, data, data_datetime):
1383 1423 f=open(filename_phase,'a')
1384 1424 timetuple_data = data_datetime.timetuple()
1385 1425 day = str(timetuple_data.tm_mday)
1386 1426 month = str(timetuple_data.tm_mon)
1387 1427 year = str(timetuple_data.tm_year)
1388 1428 hour = str(timetuple_data.tm_hour)
1389 1429 minute = str(timetuple_data.tm_min)
1390 1430 second = str(timetuple_data.tm_sec)
1391 1431 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1392 1432 f.close()
1393 1433
1394 1434
1395 1435 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1396 1436 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
1397 1437 timerange=None,
1398 1438 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1399 1439 server=None, folder=None, username=None, password=None,
1400 1440 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1401 1441
1442 if dataOut.flagNoData:
1443 return dataOut
1444
1402 1445 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1403 1446 return
1404 1447
1405 1448 if pairsList == None:
1406 1449 pairsIndexList = dataOut.pairsIndexList[:10]
1407 1450 else:
1408 1451 pairsIndexList = []
1409 1452 for pair in pairsList:
1410 1453 if pair not in dataOut.pairsList:
1411 1454 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
1412 1455 pairsIndexList.append(dataOut.pairsList.index(pair))
1413 1456
1414 1457 if pairsIndexList == []:
1415 1458 return
1416 1459
1417 1460 # if len(pairsIndexList) > 4:
1418 1461 # pairsIndexList = pairsIndexList[0:4]
1419 1462
1420 1463 hmin_index = None
1421 1464 hmax_index = None
1422 1465
1423 1466 if hmin != None and hmax != None:
1424 1467 indexes = numpy.arange(dataOut.nHeights)
1425 1468 hmin_list = indexes[dataOut.heightList >= hmin]
1426 1469 hmax_list = indexes[dataOut.heightList <= hmax]
1427 1470
1428 1471 if hmin_list.any():
1429 1472 hmin_index = hmin_list[0]
1430 1473
1431 1474 if hmax_list.any():
1432 1475 hmax_index = hmax_list[-1]+1
1433 1476
1434 1477 x = dataOut.getTimeRange()
1435 1478 #y = dataOut.getHeiRange()
1436 1479
1437 1480
1438 1481 thisDatetime = dataOut.datatime
1439 1482
1440 1483 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1441 1484 xlabel = "Local Time"
1442 1485 ylabel = "Phase (degrees)"
1443 1486
1444 1487 update_figfile = False
1445 1488
1446 1489 nplots = len(pairsIndexList)
1447 1490 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1448 1491 phase_beacon = numpy.zeros(len(pairsIndexList))
1449 1492 for i in range(nplots):
1450 1493 pair = dataOut.pairsList[pairsIndexList[i]]
1451 1494 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
1452 1495 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
1453 1496 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
1454 1497 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
1455 1498 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
1456 1499
1457 1500 #print "Phase %d%d" %(pair[0], pair[1])
1458 1501 #print phase[dataOut.beacon_heiIndexList]
1459 1502
1460 1503 if dataOut.beacon_heiIndexList:
1461 1504 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
1462 1505 else:
1463 1506 phase_beacon[i] = numpy.average(phase)
1464 1507
1465 1508 if not self.isConfig:
1466 1509
1467 1510 nplots = len(pairsIndexList)
1468 1511
1469 1512 self.setup(id=id,
1470 1513 nplots=nplots,
1471 1514 wintitle=wintitle,
1472 1515 showprofile=showprofile,
1473 1516 show=show)
1474 1517
1475 1518 if timerange != None:
1476 1519 self.timerange = timerange
1477 1520
1478 1521 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1479 1522
1480 1523 if ymin == None: ymin = 0
1481 1524 if ymax == None: ymax = 360
1482 1525
1483 1526 self.FTP_WEI = ftp_wei
1484 1527 self.EXP_CODE = exp_code
1485 1528 self.SUB_EXP_CODE = sub_exp_code
1486 1529 self.PLOT_POS = plot_pos
1487 1530
1488 1531 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1489 1532 self.isConfig = True
1490 1533 self.figfile = figfile
1491 1534 self.xdata = numpy.array([])
1492 1535 self.ydata = numpy.array([])
1493 1536
1494 1537 update_figfile = True
1495 1538
1496 1539 #open file beacon phase
1497 1540 path = '%s%03d' %(self.PREFIX, self.id)
1498 1541 beacon_file = os.path.join(path,'%s.txt'%self.name)
1499 1542 self.filename_phase = os.path.join(figpath,beacon_file)
1500 1543 #self.save_phase(self.filename_phase)
1501 1544
1502 1545
1503 1546 #store data beacon phase
1504 1547 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1505 1548
1506 1549 self.setWinTitle(title)
1507 1550
1508 1551
1509 1552 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1510 1553
1511 1554 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
1512 1555
1513 1556 axes = self.axesList[0]
1514 1557
1515 1558 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1516 1559
1517 1560 if len(self.ydata)==0:
1518 1561 self.ydata = phase_beacon.reshape(-1,1)
1519 1562 else:
1520 1563 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1521 1564
1522 1565
1523 1566 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1524 1567 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1525 1568 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1526 1569 XAxisAsTime=True, grid='both'
1527 1570 )
1528 1571
1529 1572 self.draw()
1530 1573
1531 1574 if dataOut.ltctime >= self.xmax:
1532 1575 self.counter_imagwr = wr_period
1533 1576 self.isConfig = False
1534 1577 update_figfile = True
1535 1578
1536 1579 self.save(figpath=figpath,
1537 1580 figfile=figfile,
1538 1581 save=save,
1539 1582 ftp=ftp,
1540 1583 wr_period=wr_period,
1541 1584 thisDatetime=thisDatetime,
1542 update_figfile=update_figfile) No newline at end of file
1585 update_figfile=update_figfile)
1586
1587 return dataOut #Yong No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1826 +1,1826
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 import schainpy.admin
24 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
25 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
26 25 from schainpy.utils import log
27 26 import schainpy.admin
28 27
29 28 LOCALTIME = True
30 29
31 30
32 31 def isNumber(cad):
33 32 """
34 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
35 34
36 35 Excepciones:
37 36 Si un determinado string no puede ser convertido a numero
38 37 Input:
39 38 str, string al cual se le analiza para determinar si convertible a un numero o no
40 39
41 40 Return:
42 41 True : si el string es uno numerico
43 42 False : no es un string numerico
44 43 """
45 44 try:
46 45 float(cad)
47 46 return True
48 47 except:
49 48 return False
50 49
51 50
52 51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
53 52 """
54 53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
55 54
56 55 Inputs:
57 56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
58 57
59 58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
60 59 segundos contados desde 01/01/1970.
61 60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
62 61 segundos contados desde 01/01/1970.
63 62
64 63 Return:
65 64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
66 65 fecha especificado, de lo contrario retorna False.
67 66
68 67 Excepciones:
69 68 Si el archivo no existe o no puede ser abierto
70 69 Si la cabecera no puede ser leida.
71 70
72 71 """
73 72 basicHeaderObj = BasicHeader(LOCALTIME)
74 73
75 74 try:
76 75 fp = open(filename, 'rb')
77 76 except IOError:
78 77 print("The file %s can't be opened" % (filename))
79 78 return 0
80 79
81 80 sts = basicHeaderObj.read(fp)
82 81 fp.close()
83 82
84 83 if not(sts):
85 84 print("Skipping the file %s because it has not a valid header" % (filename))
86 85 return 0
87 86
88 87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
89 88 return 0
90 89
91 90 return 1
92 91
93 92
94 93 def isTimeInRange(thisTime, startTime, endTime):
95 94 if endTime >= startTime:
96 95 if (thisTime < startTime) or (thisTime > endTime):
97 96 return 0
98 97 return 1
99 98 else:
100 99 if (thisTime < startTime) and (thisTime > endTime):
101 100 return 0
102 101 return 1
103 102
104 103
105 104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
106 105 """
107 106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
108 107
109 108 Inputs:
110 109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
111 110
112 111 startDate : fecha inicial del rango seleccionado en formato datetime.date
113 112
114 113 endDate : fecha final del rango seleccionado en formato datetime.date
115 114
116 115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
117 116
118 117 endTime : tiempo final del rango seleccionado en formato datetime.time
119 118
120 119 Return:
121 120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
122 121 fecha especificado, de lo contrario retorna False.
123 122
124 123 Excepciones:
125 124 Si el archivo no existe o no puede ser abierto
126 125 Si la cabecera no puede ser leida.
127 126
128 127 """
129 128
130 129 try:
131 130 fp = open(filename, 'rb')
132 131 except IOError:
133 132 print("The file %s can't be opened" % (filename))
134 133 return None
135 134
136 135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
137 136 systemHeaderObj = SystemHeader()
138 137 radarControllerHeaderObj = RadarControllerHeader()
139 138 processingHeaderObj = ProcessingHeader()
140 139
141 140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
142 141
143 142 sts = firstBasicHeaderObj.read(fp)
144 143
145 144 if not(sts):
146 145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
147 146 return None
148 147
149 148 if not systemHeaderObj.read(fp):
150 149 return None
151 150
152 151 if not radarControllerHeaderObj.read(fp):
153 152 return None
154 153
155 154 if not processingHeaderObj.read(fp):
156 155 return None
157 156
158 157 filesize = os.path.getsize(filename)
159 158
160 159 offset = processingHeaderObj.blockSize + 24 # header size
161 160
162 161 if filesize <= offset:
163 162 print("[Reading] %s: This file has not enough data" % filename)
164 163 return None
165 164
166 165 fp.seek(-offset, 2)
167 166
168 167 sts = lastBasicHeaderObj.read(fp)
169 168
170 169 fp.close()
171 170
172 171 thisDatetime = lastBasicHeaderObj.datatime
173 172 thisTime_last_block = thisDatetime.time()
174 173
175 174 thisDatetime = firstBasicHeaderObj.datatime
176 175 thisDate = thisDatetime.date()
177 176 thisTime_first_block = thisDatetime.time()
178 177
179 178 # General case
180 179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
181 180 #-----------o----------------------------o-----------
182 181 # startTime endTime
183 182
184 183 if endTime >= startTime:
185 184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
186 185 return None
187 186
188 187 return thisDatetime
189 188
190 189 # If endTime < startTime then endTime belongs to the next day
191 190
192 191 #<<<<<<<<<<<o o>>>>>>>>>>>
193 192 #-----------o----------------------------o-----------
194 193 # endTime startTime
195 194
196 195 if (thisDate == startDate) and (thisTime_last_block < startTime):
197 196 return None
198 197
199 198 if (thisDate == endDate) and (thisTime_first_block > endTime):
200 199 return None
201 200
202 201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
203 202 return None
204 203
205 204 return thisDatetime
206 205
207 206
208 207 def isFolderInDateRange(folder, startDate=None, endDate=None):
209 208 """
210 209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
211 210
212 211 Inputs:
213 212 folder : nombre completo del directorio.
214 213 Su formato deberia ser "/path_root/?YYYYDDD"
215 214
216 215 siendo:
217 216 YYYY : Anio (ejemplo 2015)
218 217 DDD : Dia del anio (ejemplo 305)
219 218
220 219 startDate : fecha inicial del rango seleccionado en formato datetime.date
221 220
222 221 endDate : fecha final del rango seleccionado en formato datetime.date
223 222
224 223 Return:
225 224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
226 225 fecha especificado, de lo contrario retorna False.
227 226 Excepciones:
228 227 Si el directorio no tiene el formato adecuado
229 228 """
230 229
231 230 basename = os.path.basename(folder)
232 231
233 232 if not isRadarFolder(basename):
234 233 print("The folder %s has not the rigth format" % folder)
235 234 return 0
236 235
237 236 if startDate and endDate:
238 237 thisDate = getDateFromRadarFolder(basename)
239 238
240 239 if thisDate < startDate:
241 240 return 0
242 241
243 242 if thisDate > endDate:
244 243 return 0
245 244
246 245 return 1
247 246
248 247
249 248 def isFileInDateRange(filename, startDate=None, endDate=None):
250 249 """
251 250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
252 251
253 252 Inputs:
254 253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
255 254
256 255 Su formato deberia ser "?YYYYDDDsss"
257 256
258 257 siendo:
259 258 YYYY : Anio (ejemplo 2015)
260 259 DDD : Dia del anio (ejemplo 305)
261 260 sss : set
262 261
263 262 startDate : fecha inicial del rango seleccionado en formato datetime.date
264 263
265 264 endDate : fecha final del rango seleccionado en formato datetime.date
266 265
267 266 Return:
268 267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
269 268 fecha especificado, de lo contrario retorna False.
270 269 Excepciones:
271 270 Si el archivo no tiene el formato adecuado
272 271 """
273 272
274 273 basename = os.path.basename(filename)
275 274
276 275 if not isRadarFile(basename):
277 276 print("The filename %s has not the rigth format" % filename)
278 277 return 0
279 278
280 279 if startDate and endDate:
281 280 thisDate = getDateFromRadarFile(basename)
282 281
283 282 if thisDate < startDate:
284 283 return 0
285 284
286 285 if thisDate > endDate:
287 286 return 0
288 287
289 288 return 1
290 289
291 290
292 291 def getFileFromSet(path, ext, set):
293 292 validFilelist = []
294 293 fileList = os.listdir(path)
295 294
296 295 # 0 1234 567 89A BCDE
297 296 # H YYYY DDD SSS .ext
298 297
299 298 for thisFile in fileList:
300 299 try:
301 300 year = int(thisFile[1:5])
302 301 doy = int(thisFile[5:8])
303 302 except:
304 303 continue
305 304
306 305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 306 continue
308 307
309 308 validFilelist.append(thisFile)
310 309
311 310 myfile = fnmatch.filter(
312 311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
313 312
314 313 if len(myfile) != 0:
315 314 return myfile[0]
316 315 else:
317 316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
318 317 print('the filename %s does not exist' % filename)
319 318 print('...going to the last file: ')
320 319
321 320 if validFilelist:
322 321 validFilelist = sorted(validFilelist, key=str.lower)
323 322 return validFilelist[-1]
324 323
325 324 return None
326 325
327 326
328 327 def getlastFileFromPath(path, ext):
329 328 """
330 329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
331 330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
332 331
333 332 Input:
334 333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
335 334 ext : extension de los files contenidos en una carpeta
336 335
337 336 Return:
338 337 El ultimo file de una determinada carpeta, no se considera el path.
339 338 """
340 339 validFilelist = []
341 340 fileList = os.listdir(path)
342 341
343 342 # 0 1234 567 89A BCDE
344 343 # H YYYY DDD SSS .ext
345 344
346 345 for thisFile in fileList:
347 346
348 347 year = thisFile[1:5]
349 348 if not isNumber(year):
350 349 continue
351 350
352 351 doy = thisFile[5:8]
353 352 if not isNumber(doy):
354 353 continue
355 354
356 355 year = int(year)
357 356 doy = int(doy)
358 357
359 358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
360 359 continue
361 360
362 361 validFilelist.append(thisFile)
363 362
364 363 if validFilelist:
365 364 validFilelist = sorted(validFilelist, key=str.lower)
366 365 return validFilelist[-1]
367 366
368 367 return None
369 368
370 369
371 370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
372 371 """
373 372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
374 373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
375 374 el path exacto de un determinado file.
376 375
377 376 Example :
378 377 nombre correcto del file es .../.../D2009307/P2009307367.ext
379 378
380 379 Entonces la funcion prueba con las siguientes combinaciones
381 380 .../.../y2009307367.ext
382 381 .../.../Y2009307367.ext
383 382 .../.../x2009307/y2009307367.ext
384 383 .../.../x2009307/Y2009307367.ext
385 384 .../.../X2009307/y2009307367.ext
386 385 .../.../X2009307/Y2009307367.ext
387 386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
388 387
389 388 Return:
390 389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
391 390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
392 391 para el filename
393 392 """
394 393 fullfilename = None
395 394 find_flag = False
396 395 filename = None
397 396
398 397 prefixDirList = [None, 'd', 'D']
399 398 if ext.lower() == ".r": # voltage
400 399 prefixFileList = ['d', 'D']
401 400 elif ext.lower() == ".pdata": # spectra
402 401 prefixFileList = ['p', 'P']
403 402 else:
404 403 return None, filename
405 404
406 405 # barrido por las combinaciones posibles
407 406 for prefixDir in prefixDirList:
408 407 thispath = path
409 408 if prefixDir != None:
410 409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
411 410 if foldercounter == 0:
412 411 thispath = os.path.join(path, "%s%04d%03d" %
413 412 (prefixDir, year, doy))
414 413 else:
415 414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
416 415 prefixDir, year, doy, foldercounter))
417 416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
418 417 # formo el nombre del file xYYYYDDDSSS.ext
419 418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
420 419 fullfilename = os.path.join(
421 420 thispath, filename) # formo el path completo
422 421
423 422 if os.path.exists(fullfilename): # verifico que exista
424 423 find_flag = True
425 424 break
426 425 if find_flag:
427 426 break
428 427
429 428 if not(find_flag):
430 429 return None, filename
431 430
432 431 return fullfilename, filename
433 432
434 433
435 434 def isRadarFolder(folder):
436 435 try:
437 436 year = int(folder[1:5])
438 437 doy = int(folder[5:8])
439 438 except:
440 439 return 0
441 440
442 441 return 1
443 442
444 443
445 444 def isRadarFile(file):
446 445 try:
447 446 year = int(file[1:5])
448 447 doy = int(file[5:8])
449 448 set = int(file[8:11])
450 449 except:
451 450 return 0
452 451
453 452 return 1
454 453
455 454
456 455 def getDateFromRadarFile(file):
457 456 try:
458 457 year = int(file[1:5])
459 458 doy = int(file[5:8])
460 459 set = int(file[8:11])
461 460 except:
462 461 return None
463 462
464 463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
465 464 return thisDate
466 465
467 466
468 467 def getDateFromRadarFolder(folder):
469 468 try:
470 469 year = int(folder[1:5])
471 470 doy = int(folder[5:8])
472 471 except:
473 472 return None
474 473
475 474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
476 475 return thisDate
477 476
478 477
479 478 class JRODataIO:
480 479
481 480 c = 3E8
482 481
483 482 isConfig = False
484 483
485 484 basicHeaderObj = None
486 485
487 486 systemHeaderObj = None
488 487
489 488 radarControllerHeaderObj = None
490 489
491 490 processingHeaderObj = None
492 491
493 492 dtype = None
494 493
495 494 pathList = []
496 495
497 496 filenameList = []
498 497
499 498 filename = None
500 499
501 500 ext = None
502 501
503 502 flagIsNewFile = 1
504 503
505 504 flagDiscontinuousBlock = 0
506 505
507 506 flagIsNewBlock = 0
508 507
509 508 fp = None
510 509
511 510 firstHeaderSize = 0
512 511
513 512 basicHeaderSize = 24
514 513
515 514 versionFile = 1103
516 515
517 516 fileSize = None
518 517
519 518 # ippSeconds = None
520 519
521 520 fileSizeByHeader = None
522 521
523 522 fileIndex = None
524 523
525 524 profileIndex = None
526 525
527 526 blockIndex = None
528 527
529 528 nTotalBlocks = None
530 529
531 530 maxTimeStep = 30
532 531
533 532 lastUTTime = None
534 533
535 534 datablock = None
536 535
537 536 dataOut = None
538 537
539 538 blocksize = None
540 539
541 540 getByBlock = False
542 541
543 542 def __init__(self):
544 543
545 544 raise NotImplementedError
546 545
547 546 def run(self):
548 547
549 548 raise NotImplementedError
550 549
551 550 def getDtypeWidth(self):
552 551
553 552 dtype_index = get_dtype_index(self.dtype)
554 553 dtype_width = get_dtype_width(dtype_index)
555 554
556 555 return dtype_width
557 556
558 557 def getAllowedArgs(self):
559 558 if hasattr(self, '__attrs__'):
560 559 return self.__attrs__
561 560 else:
562 561 return inspect.getargspec(self.run).args
563 562
564 563
565 564 class JRODataReader(JRODataIO):
566 565
567 566 online = 0
568 567
569 568 realtime = 0
570 569
571 570 nReadBlocks = 0
572 571
573 572 delay = 10 # number of seconds waiting a new file
574 573
575 574 nTries = 3 # quantity tries
576 575
577 576 nFiles = 3 # number of files for searching
578 577
579 578 path = None
580 579
581 580 foldercounter = 0
582 581
583 582 flagNoMoreFiles = 0
584 583
585 584 datetimeList = []
586 585
587 586 __isFirstTimeOnline = 1
588 587
589 588 __printInfo = True
590 589
591 590 profileIndex = None
592 591
593 592 nTxs = 1
594 593
595 594 txIndex = None
596 595
597 596 # Added--------------------
598 597
599 598 selBlocksize = None
600 599
601 600 selBlocktime = None
602 601
603 602 def __init__(self):
604 603 """
605 604 This class is used to find data files
606 605
607 606 Example:
608 607 reader = JRODataReader()
609 608 fileList = reader.findDataFiles()
610 609
611 610 """
612 611 pass
613 612
614 613 def createObjByDefault(self):
615 614 """
616 615
617 616 """
618 617 raise NotImplementedError
619 618
620 619 def getBlockDimension(self):
621 620
622 621 raise NotImplementedError
623 622
624 623 def searchFilesOffLine(self,
625 624 path,
626 625 startDate=None,
627 626 endDate=None,
628 627 startTime=datetime.time(0, 0, 0),
629 628 endTime=datetime.time(23, 59, 59),
630 629 set=None,
631 630 expLabel='',
632 631 ext='.r',
633 632 cursor=None,
634 633 skip=None,
635 634 walk=True):
636 635
637 636 self.filenameList = []
638 637 self.datetimeList = []
639 638
640 639 pathList = []
641 640
642 641 dateList, pathList = self.findDatafiles(
643 642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
644 643
645 644 if dateList == []:
646 645 return [], []
647 646
648 647 if len(dateList) > 1:
649 648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
650 649 else:
651 650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
652 651
653 652 filenameList = []
654 653 datetimeList = []
655 654
656 655 for thisPath in pathList:
657 656
658 657 fileList = glob.glob1(thisPath, "*%s" % ext)
659 658 fileList.sort()
660 659
661 660 for file in fileList:
662 661
663 662 filename = os.path.join(thisPath, file)
664 663
665 664 if not isFileInDateRange(filename, startDate, endDate):
666 665 continue
667 666
668 667 thisDatetime = isFileInTimeRange(
669 668 filename, startDate, endDate, startTime, endTime)
670 669
671 670 if not(thisDatetime):
672 671 continue
673 672
674 673 filenameList.append(filename)
675 674 datetimeList.append(thisDatetime)
676 675
677 676 if cursor is not None and skip is not None:
678 677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
679 678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
680 679
681 680 if not(filenameList):
682 681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
683 682 return [], []
684 683
685 684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
686 685
687 686 # for i in range(len(filenameList)):
688 687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
689 688
690 689 self.filenameList = filenameList
691 690 self.datetimeList = datetimeList
692 691
693 692 return pathList, filenameList
694 693
695 694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
696 695 """
697 696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
698 697 devuelve el archivo encontrado ademas de otros datos.
699 698
700 699 Input:
701 700 path : carpeta donde estan contenidos los files que contiene data
702 701
703 702 expLabel : Nombre del subexperimento (subfolder)
704 703
705 704 ext : extension de los files
706 705
707 706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
708 707
709 708 Return:
710 709 directory : eL directorio donde esta el file encontrado
711 710 filename : el ultimo file de una determinada carpeta
712 711 year : el anho
713 712 doy : el numero de dia del anho
714 713 set : el set del archivo
715 714
716 715
717 716 """
718 717 if not os.path.isdir(path):
719 718 return None, None, None, None, None, None
720 719
721 720 dirList = []
722 721
723 722 if not walk:
724 723 fullpath = path
725 724 foldercounter = 0
726 725 else:
727 726 # Filtra solo los directorios
728 727 for thisPath in os.listdir(path):
729 728 if not os.path.isdir(os.path.join(path, thisPath)):
730 729 continue
731 730 if not isRadarFolder(thisPath):
732 731 continue
733 732
734 733 dirList.append(thisPath)
735 734
736 735 if not(dirList):
737 736 return None, None, None, None, None, None
738 737
739 738 dirList = sorted(dirList, key=str.lower)
740 739
741 740 doypath = dirList[-1]
742 741 foldercounter = int(doypath.split('_')[1]) if len(
743 742 doypath.split('_')) > 1 else 0
744 743 fullpath = os.path.join(path, doypath, expLabel)
745 744
746 745 print("[Reading] %s folder was found: " % (fullpath))
747 746
748 747 if set == None:
749 748 filename = getlastFileFromPath(fullpath, ext)
750 749 else:
751 750 filename = getFileFromSet(fullpath, ext, set)
752 751
753 752 if not(filename):
754 753 return None, None, None, None, None, None
755 754
756 755 print("[Reading] %s file was found" % (filename))
757 756
758 757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
759 758 return None, None, None, None, None, None
760 759
761 760 year = int(filename[1:5])
762 761 doy = int(filename[5:8])
763 762 set = int(filename[8:11])
764 763
765 764 return fullpath, foldercounter, filename, year, doy, set
766 765
767 766 def __setNextFileOffline(self):
768 767
769 768 idFile = self.fileIndex
770 769
771 770 while (True):
772 771 idFile += 1
773 772 if not(idFile < len(self.filenameList)):
774 773 self.flagNoMoreFiles = 1
775 774 # print "[Reading] No more Files"
776 775 return 0
777 776
778 777 filename = self.filenameList[idFile]
779 778
780 779 if not(self.__verifyFile(filename)):
781 780 continue
782 781
783 782 fileSize = os.path.getsize(filename)
784 783 fp = open(filename, 'rb')
785 784 break
786 785
787 786 self.flagIsNewFile = 1
788 787 self.fileIndex = idFile
789 788 self.filename = filename
790 789 self.fileSize = fileSize
791 790 self.fp = fp
792 791
793 792 # print "[Reading] Setting the file: %s"%self.filename
794 793
795 794 return 1
796 795
797 796 def __setNextFileOnline(self):
798 797 """
799 798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
800 799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
801 800 siguientes.
802 801
803 802 Affected:
804 803 self.flagIsNewFile
805 804 self.filename
806 805 self.fileSize
807 806 self.fp
808 807 self.set
809 808 self.flagNoMoreFiles
810 809
811 810 Return:
812 811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
813 812 1 : si el file fue abierto con exito y esta listo a ser leido
814 813
815 814 Excepciones:
816 815 Si un determinado file no puede ser abierto
817 816 """
818 817 nFiles = 0
819 818 fileOk_flag = False
820 819 firstTime_flag = True
821 820
822 821 self.set += 1
823 822
824 823 if self.set > 999:
825 824 self.set = 0
826 825 self.foldercounter += 1
827 826
828 827 # busca el 1er file disponible
829 828 fullfilename, filename = checkForRealPath(
830 829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
831 830 if fullfilename:
832 831 if self.__verifyFile(fullfilename, False):
833 832 fileOk_flag = True
834 833
835 834 # si no encuentra un file entonces espera y vuelve a buscar
836 835 if not(fileOk_flag):
837 836 # busco en los siguientes self.nFiles+1 files posibles
838 837 for nFiles in range(self.nFiles + 1):
839 838
840 839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
841 840 tries = self.nTries
842 841 else:
843 842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
844 843
845 844 for nTries in range(tries):
846 845 if firstTime_flag:
847 846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
848 847 sleep(self.delay)
849 848 else:
850 849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
851 850
852 851 fullfilename, filename = checkForRealPath(
853 852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
854 853 if fullfilename:
855 854 if self.__verifyFile(fullfilename):
856 855 fileOk_flag = True
857 856 break
858 857
859 858 if fileOk_flag:
860 859 break
861 860
862 861 firstTime_flag = False
863 862
864 863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
865 864 self.set += 1
866 865
867 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
868 867 if nFiles == (self.nFiles - 1):
869 868 self.set = 0
870 869 self.doy += 1
871 870 self.foldercounter = 0
872 871
873 872 if fileOk_flag:
874 873 self.fileSize = os.path.getsize(fullfilename)
875 874 self.filename = fullfilename
876 875 self.flagIsNewFile = 1
877 876 if self.fp != None:
878 877 self.fp.close()
879 878 self.fp = open(fullfilename, 'rb')
880 879 self.flagNoMoreFiles = 0
881 880 # print '[Reading] Setting the file: %s' % fullfilename
882 881 else:
883 882 self.fileSize = 0
884 883 self.filename = None
885 884 self.flagIsNewFile = 0
886 885 self.fp = None
887 886 self.flagNoMoreFiles = 1
887 # print '[Reading] No more files to read'
888 888
889 889 return fileOk_flag
890 890
891 891 def setNextFile(self):
892 892 if self.fp != None:
893 893 self.fp.close()
894 894
895 895 if self.online:
896 896 newFile = self.__setNextFileOnline()
897 897 else:
898 898 newFile = self.__setNextFileOffline()
899 899
900 if not(newFile):
901 raise schainpy.admin.SchainWarning('No more files to read')
900 if not(newFile):
901 raise(schainpy.admin.SchainWarning('No more files to read'))
902 902 return 0
903 903
904 904 if self.verbose:
905 905 print('[Reading] Setting the file: %s' % self.filename)
906 906
907 907 self.__readFirstHeader()
908 908 self.nReadBlocks = 0
909 909 return 1
910 910
911 911 def __waitNewBlock(self):
912 912 """
913 913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914 914
915 915 Si el modo de lectura es OffLine siempre retorn 0
916 916 """
917 917 if not self.online:
918 918 return 0
919 919
920 920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 921 return 0
922 922
923 923 currentPointer = self.fp.tell()
924 924
925 925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926 926
927 927 for nTries in range(self.nTries):
928 928
929 929 self.fp.close()
930 930 self.fp = open(self.filename, 'rb')
931 931 self.fp.seek(currentPointer)
932 932
933 933 self.fileSize = os.path.getsize(self.filename)
934 934 currentSize = self.fileSize - currentPointer
935 935
936 936 if (currentSize >= neededSize):
937 937 self.basicHeaderObj.read(self.fp)
938 938 return 1
939 939
940 940 if self.fileSize == self.fileSizeByHeader:
941 941 # self.flagEoF = True
942 942 return 0
943 943
944 944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 945 sleep(self.delay)
946 946
947 947 return 0
948 948
949 949 def waitDataBlock(self, pointer_location):
950 950
951 951 currentPointer = pointer_location
952 952
953 953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 954
955 955 for nTries in range(self.nTries):
956 956 self.fp.close()
957 957 self.fp = open(self.filename, 'rb')
958 958 self.fp.seek(currentPointer)
959 959
960 960 self.fileSize = os.path.getsize(self.filename)
961 961 currentSize = self.fileSize - currentPointer
962 962
963 963 if (currentSize >= neededSize):
964 964 return 1
965 965
966 966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
967 967 sleep(self.delay)
968 968
969 969 return 0
970 970
971 971 def __jumpToLastBlock(self):
972 972
973 973 if not(self.__isFirstTimeOnline):
974 974 return
975 975
976 976 csize = self.fileSize - self.fp.tell()
977 977 blocksize = self.processingHeaderObj.blockSize
978 978
979 979 # salta el primer bloque de datos
980 980 if csize > self.processingHeaderObj.blockSize:
981 981 self.fp.seek(self.fp.tell() + blocksize)
982 982 else:
983 983 return
984 984
985 985 csize = self.fileSize - self.fp.tell()
986 986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
987 987 while True:
988 988
989 989 if self.fp.tell() < self.fileSize:
990 990 self.fp.seek(self.fp.tell() + neededsize)
991 991 else:
992 992 self.fp.seek(self.fp.tell() - neededsize)
993 993 break
994 994
995 995 # csize = self.fileSize - self.fp.tell()
996 996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
997 997 # factor = int(csize/neededsize)
998 998 # if factor > 0:
999 999 # self.fp.seek(self.fp.tell() + factor*neededsize)
1000 1000
1001 1001 self.flagIsNewFile = 0
1002 1002 self.__isFirstTimeOnline = 0
1003 1003
1004 1004 def __setNewBlock(self):
1005 1005 # if self.server is None:
1006 1006 if self.fp == None:
1007 1007 return 0
1008 1008
1009 1009 # if self.online:
1010 1010 # self.__jumpToLastBlock()
1011 1011
1012 1012 if self.flagIsNewFile:
1013 1013 self.lastUTTime = self.basicHeaderObj.utc
1014 1014 return 1
1015 1015
1016 1016 if self.realtime:
1017 1017 self.flagDiscontinuousBlock = 1
1018 1018 if not(self.setNextFile()):
1019 1019 return 0
1020 1020 else:
1021 1021 return 1
1022 1022 # if self.server is None:
1023 1023 currentSize = self.fileSize - self.fp.tell()
1024 1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1025 1025 if (currentSize >= neededSize):
1026 1026 self.basicHeaderObj.read(self.fp)
1027 1027 self.lastUTTime = self.basicHeaderObj.utc
1028 1028 return 1
1029 1029 # else:
1030 1030 # self.basicHeaderObj.read(self.zHeader)
1031 1031 # self.lastUTTime = self.basicHeaderObj.utc
1032 1032 # return 1
1033 1033 if self.__waitNewBlock():
1034 1034 self.lastUTTime = self.basicHeaderObj.utc
1035 1035 return 1
1036 1036 # if self.server is None:
1037 1037 if not(self.setNextFile()):
1038 1038 return 0
1039 1039
1040 1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1041 1041 self.lastUTTime = self.basicHeaderObj.utc
1042 1042
1043 1043 self.flagDiscontinuousBlock = 0
1044 1044
1045 1045 if deltaTime > self.maxTimeStep:
1046 1046 self.flagDiscontinuousBlock = 1
1047 1047
1048 1048 return 1
1049 1049
1050 1050 def readNextBlock(self):
1051 1051
1052 1052 # Skip block out of startTime and endTime
1053 1053 while True:
1054 1054 if not(self.__setNewBlock()):
1055 raise schainpy
1055 raise(schainpy.admin.SchainWarning('No more files'))
1056 1056 return 0
1057 1057
1058 1058 if not(self.readBlock()):
1059 1059 return 0
1060 1060
1061 1061 self.getBasicHeader()
1062 1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1063 1063 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1064 1064 self.processingHeaderObj.dataBlocksPerFile,
1065 1065 self.dataOut.datatime.ctime()))
1066 1066 continue
1067 1067
1068 1068 break
1069 1069
1070 1070 if self.verbose:
1071 1071 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1072 1072 self.processingHeaderObj.dataBlocksPerFile,
1073 1073 self.dataOut.datatime.ctime()))
1074 1074 return 1
1075 1075
1076 1076 def __readFirstHeader(self):
1077 1077
1078 1078 self.basicHeaderObj.read(self.fp)
1079 1079 self.systemHeaderObj.read(self.fp)
1080 1080 self.radarControllerHeaderObj.read(self.fp)
1081 1081 self.processingHeaderObj.read(self.fp)
1082 1082
1083 1083 self.firstHeaderSize = self.basicHeaderObj.size
1084 1084
1085 1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1086 1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1087 1087 if datatype == 0:
1088 1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1089 1089 elif datatype == 1:
1090 1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1091 1091 elif datatype == 2:
1092 1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1093 1093 elif datatype == 3:
1094 1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1095 1095 elif datatype == 4:
1096 1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1097 1097 elif datatype == 5:
1098 1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1099 1099 else:
1100 1100 raise ValueError('Data type was not defined')
1101 1101
1102 1102 self.dtype = datatype_str
1103 1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1104 1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1105 1105 self.firstHeaderSize + self.basicHeaderSize * \
1106 1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1107 1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1108 1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1109 1109 self.getBlockDimension()
1110 1110
1111 1111 def __verifyFile(self, filename, msgFlag=True):
1112 1112
1113 1113 msg = None
1114 1114
1115 1115 try:
1116 1116 fp = open(filename, 'rb')
1117 1117 except IOError:
1118 1118
1119 1119 if msgFlag:
1120 1120 print("[Reading] File %s can't be opened" % (filename))
1121 1121
1122 1122 return False
1123 1123
1124 1124 currentPosition = fp.tell()
1125 1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1126 1126
1127 1127 if neededSize == 0:
1128 1128 basicHeaderObj = BasicHeader(LOCALTIME)
1129 1129 systemHeaderObj = SystemHeader()
1130 1130 radarControllerHeaderObj = RadarControllerHeader()
1131 1131 processingHeaderObj = ProcessingHeader()
1132 1132
1133 1133 if not(basicHeaderObj.read(fp)):
1134 1134 fp.close()
1135 1135 return False
1136 1136
1137 1137 if not(systemHeaderObj.read(fp)):
1138 1138 fp.close()
1139 1139 return False
1140 1140
1141 1141 if not(radarControllerHeaderObj.read(fp)):
1142 1142 fp.close()
1143 1143 return False
1144 1144
1145 1145 if not(processingHeaderObj.read(fp)):
1146 1146 fp.close()
1147 1147 return False
1148 1148
1149 1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1150 1150 else:
1151 1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1152 1152
1153 1153 fp.close()
1154 1154
1155 1155 fileSize = os.path.getsize(filename)
1156 1156 currentSize = fileSize - currentPosition
1157 1157
1158 1158 if currentSize < neededSize:
1159 1159 if msgFlag and (msg != None):
1160 1160 print(msg)
1161 1161 return False
1162 1162
1163 1163 return True
1164 1164
1165 1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1166 1166
1167 1167 path_empty = True
1168 1168
1169 1169 dateList = []
1170 1170 pathList = []
1171 1171
1172 1172 multi_path = path.split(',')
1173 1173
1174 1174 if not walk:
1175 1175
1176 1176 for single_path in multi_path:
1177 1177
1178 1178 if not os.path.isdir(single_path):
1179 1179 continue
1180 1180
1181 1181 fileList = glob.glob1(single_path, "*" + ext)
1182 1182
1183 1183 if not fileList:
1184 1184 continue
1185 1185
1186 1186 path_empty = False
1187 1187
1188 1188 fileList.sort()
1189 1189
1190 1190 for thisFile in fileList:
1191 1191
1192 1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1193 1193 continue
1194 1194
1195 1195 if not isRadarFile(thisFile):
1196 1196 continue
1197 1197
1198 1198 if not isFileInDateRange(thisFile, startDate, endDate):
1199 1199 continue
1200 1200
1201 1201 thisDate = getDateFromRadarFile(thisFile)
1202 1202
1203 1203 if thisDate in dateList:
1204 1204 continue
1205 1205
1206 1206 dateList.append(thisDate)
1207 1207 pathList.append(single_path)
1208 1208
1209 1209 else:
1210 1210 for single_path in multi_path:
1211 1211
1212 1212 if not os.path.isdir(single_path):
1213 1213 continue
1214 1214
1215 1215 dirList = []
1216 1216
1217 1217 for thisPath in os.listdir(single_path):
1218 1218
1219 1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1220 1220 continue
1221 1221
1222 1222 if not isRadarFolder(thisPath):
1223 1223 continue
1224 1224
1225 1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1226 1226 continue
1227 1227
1228 1228 dirList.append(thisPath)
1229 1229
1230 1230 if not dirList:
1231 1231 continue
1232 1232
1233 1233 dirList.sort()
1234 1234
1235 1235 for thisDir in dirList:
1236 1236
1237 1237 datapath = os.path.join(single_path, thisDir, expLabel)
1238 1238 fileList = glob.glob1(datapath, "*" + ext)
1239 1239
1240 1240 if not fileList:
1241 1241 continue
1242 1242
1243 1243 path_empty = False
1244 1244
1245 1245 thisDate = getDateFromRadarFolder(thisDir)
1246 1246
1247 1247 pathList.append(datapath)
1248 1248 dateList.append(thisDate)
1249 1249
1250 1250 dateList.sort()
1251 1251
1252 1252 if walk:
1253 1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1254 1254 else:
1255 1255 pattern_path = multi_path[0]
1256 1256
1257 1257 if path_empty:
1258 1258 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1259 1259 else:
1260 1260 if not dateList:
1261 1261 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1262 1262
1263 1263 if include_path:
1264 1264 return dateList, pathList
1265 1265
1266 1266 return dateList
1267 1267
1268 1268 def setup(self,
1269 1269 path=None,
1270 1270 startDate=None,
1271 1271 endDate=None,
1272 1272 startTime=datetime.time(0, 0, 0),
1273 1273 endTime=datetime.time(23, 59, 59),
1274 1274 set=None,
1275 1275 expLabel="",
1276 1276 ext=None,
1277 1277 online=False,
1278 1278 delay=60,
1279 1279 walk=True,
1280 1280 getblock=False,
1281 1281 nTxs=1,
1282 1282 realtime=False,
1283 1283 blocksize=None,
1284 1284 blocktime=None,
1285 1285 skip=None,
1286 1286 cursor=None,
1287 1287 warnings=True,
1288 1288 verbose=True,
1289 1289 server=None,
1290 1290 format=None,
1291 1291 oneDDict=None,
1292 1292 twoDDict=None,
1293 1293 ind2DList=None):
1294 1294 if server is not None:
1295 1295 if 'tcp://' in server:
1296 1296 address = server
1297 1297 else:
1298 1298 address = 'ipc:///tmp/%s' % server
1299 1299 self.server = address
1300 1300 self.context = zmq.Context()
1301 1301 self.receiver = self.context.socket(zmq.PULL)
1302 1302 self.receiver.connect(self.server)
1303 1303 time.sleep(0.5)
1304 1304 print('[Starting] ReceiverData from {}'.format(self.server))
1305 1305 else:
1306 1306 self.server = None
1307 1307 if path == None:
1308 1308 raise ValueError("[Reading] The path is not valid")
1309 1309
1310 1310 if ext == None:
1311 1311 ext = self.ext
1312 1312
1313 1313 if online:
1314 1314 print("[Reading] Searching files in online mode...")
1315 1315
1316 1316 for nTries in range(self.nTries):
1317 1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1318 1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1319 1319
1320 1320 if fullpath:
1321 1321 break
1322 1322
1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (delay, path, nTries + 1))
1324 sleep(delay)
1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1324 sleep(self.delay)
1325 1325
1326 if not(fullpath):
1327 raise schainpy.admin.SchainWarning('There isn\'t any valid file in {}'.format(path))
1326 if not(fullpath):
1327 raise(schainpy.admin.SchainWarning('There isn\'t any valid file in {}'.format(path)))
1328 1328 return
1329 1329
1330 1330 self.year = year
1331 1331 self.doy = doy
1332 1332 self.set = set - 1
1333 1333 self.path = path
1334 1334 self.foldercounter = foldercounter
1335 1335 last_set = None
1336 1336 else:
1337 1337 print("[Reading] Searching files in offline mode ...")
1338 1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1339 1339 startTime=startTime, endTime=endTime,
1340 1340 set=set, expLabel=expLabel, ext=ext,
1341 1341 walk=walk, cursor=cursor,
1342 1342 skip=skip)
1343 1343
1344 1344 if not(pathList):
1345 1345 self.fileIndex = -1
1346 1346 self.pathList = []
1347 1347 self.filenameList = []
1348 1348 return
1349 1349
1350 1350 self.fileIndex = -1
1351 1351 self.pathList = pathList
1352 1352 self.filenameList = filenameList
1353 1353 file_name = os.path.basename(filenameList[-1])
1354 1354 basename, ext = os.path.splitext(file_name)
1355 1355 last_set = int(basename[-3:])
1356 1356
1357 1357 self.online = online
1358 1358 self.realtime = realtime
1359 1359 self.delay = delay
1360 1360 ext = ext.lower()
1361 1361 self.ext = ext
1362 1362 self.getByBlock = getblock
1363 1363 self.nTxs = nTxs
1364 1364 self.startTime = startTime
1365 1365 self.endTime = endTime
1366 1366 self.endDate = endDate
1367 1367 self.startDate = startDate
1368 1368 # Added-----------------
1369 1369 self.selBlocksize = blocksize
1370 1370 self.selBlocktime = blocktime
1371 1371
1372 1372 # Verbose-----------
1373 1373 self.verbose = verbose
1374 1374 self.warnings = warnings
1375 1375
1376 1376 if not(self.setNextFile()):
1377 1377 if (startDate != None) and (endDate != None):
1378 1378 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1379 1379 elif startDate != None:
1380 1380 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1381 1381 else:
1382 1382 print("[Reading] No files")
1383 1383
1384 1384 self.fileIndex = -1
1385 1385 self.pathList = []
1386 1386 self.filenameList = []
1387 1387 return
1388 1388
1389 1389 # self.getBasicHeader()
1390 1390
1391 1391 if last_set != None:
1392 1392 self.dataOut.last_block = last_set * \
1393 1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1394 1394 return
1395 1395
1396 1396 def getBasicHeader(self):
1397 1397
1398 1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1399 1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1400 1400
1401 1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1402 1402
1403 1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1404 1404
1405 1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1406 1406
1407 1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1408 1408
1409 1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1410 1410
1411 1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1412 1412
1413 1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1414 1414
1415 1415 def getFirstHeader(self):
1416 1416
1417 1417 raise NotImplementedError
1418 1418
1419 1419 def getData(self):
1420 1420
1421 1421 raise NotImplementedError
1422 1422
1423 1423 def hasNotDataInBuffer(self):
1424 1424
1425 1425 raise NotImplementedError
1426 1426
1427 1427 def readBlock(self):
1428 1428
1429 1429 raise NotImplementedError
1430 1430
1431 1431 def isEndProcess(self):
1432 1432
1433 1433 return self.flagNoMoreFiles
1434 1434
1435 1435 def printReadBlocks(self):
1436 1436
1437 1437 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1438 1438
1439 1439 def printTotalBlocks(self):
1440 1440
1441 1441 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1442 1442
1443 1443 def printNumberOfBlock(self):
1444 1444 'SPAM!'
1445 1445
1446 1446 # if self.flagIsNewBlock:
1447 1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1448 1448 # self.processingHeaderObj.dataBlocksPerFile,
1449 1449 # self.dataOut.datatime.ctime())
1450 1450
1451 1451 def printInfo(self):
1452 1452
1453 1453 if self.__printInfo == False:
1454 1454 return
1455 1455
1456 1456 self.basicHeaderObj.printInfo()
1457 1457 self.systemHeaderObj.printInfo()
1458 1458 self.radarControllerHeaderObj.printInfo()
1459 1459 self.processingHeaderObj.printInfo()
1460 1460
1461 1461 self.__printInfo = False
1462 1462
1463 1463 def run(self,
1464 1464 path=None,
1465 1465 startDate=None,
1466 1466 endDate=None,
1467 1467 startTime=datetime.time(0, 0, 0),
1468 1468 endTime=datetime.time(23, 59, 59),
1469 1469 set=None,
1470 1470 expLabel="",
1471 1471 ext=None,
1472 1472 online=False,
1473 1473 delay=60,
1474 1474 walk=True,
1475 1475 getblock=False,
1476 1476 nTxs=1,
1477 1477 realtime=False,
1478 1478 blocksize=None,
1479 1479 blocktime=None,
1480 1480 skip=None,
1481 1481 cursor=None,
1482 1482 warnings=True,
1483 1483 server=None,
1484 1484 verbose=True,
1485 1485 format=None,
1486 1486 oneDDict=None,
1487 1487 twoDDict=None,
1488 1488 ind2DList=None, **kwargs):
1489 1489
1490 1490 if not(self.isConfig):
1491 1491 self.setup(path=path,
1492 1492 startDate=startDate,
1493 1493 endDate=endDate,
1494 1494 startTime=startTime,
1495 1495 endTime=endTime,
1496 1496 set=set,
1497 1497 expLabel=expLabel,
1498 1498 ext=ext,
1499 1499 online=online,
1500 1500 delay=delay,
1501 1501 walk=walk,
1502 1502 getblock=getblock,
1503 1503 nTxs=nTxs,
1504 1504 realtime=realtime,
1505 1505 blocksize=blocksize,
1506 1506 blocktime=blocktime,
1507 1507 skip=skip,
1508 1508 cursor=cursor,
1509 1509 warnings=warnings,
1510 1510 server=server,
1511 1511 verbose=verbose,
1512 1512 format=format,
1513 1513 oneDDict=oneDDict,
1514 1514 twoDDict=twoDDict,
1515 1515 ind2DList=ind2DList)
1516 1516 self.isConfig = True
1517 1517 if server is None:
1518 1518 self.getData()
1519 1519 else:
1520 1520 self.getFromServer()
1521 1521
1522 1522
1523 1523 class JRODataWriter(JRODataIO):
1524 1524
1525 1525 """
1526 1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1527 1527 de los datos siempre se realiza por bloques.
1528 1528 """
1529 1529
1530 1530 blockIndex = 0
1531 1531
1532 1532 path = None
1533 1533
1534 1534 setFile = None
1535 1535
1536 1536 profilesPerBlock = None
1537 1537
1538 1538 blocksPerFile = None
1539 1539
1540 1540 nWriteBlocks = 0
1541 1541
1542 1542 fileDate = None
1543 1543
1544 1544 def __init__(self, dataOut=None):
1545 1545 raise NotImplementedError
1546 1546
1547 1547 def hasAllDataInBuffer(self):
1548 1548 raise NotImplementedError
1549 1549
1550 1550 def setBlockDimension(self):
1551 1551 raise NotImplementedError
1552 1552
1553 1553 def writeBlock(self):
1554 1554 raise NotImplementedError
1555 1555
1556 1556 def putData(self):
1557 1557 raise NotImplementedError
1558 1558
1559 1559 def getProcessFlags(self):
1560 1560
1561 1561 processFlags = 0
1562 1562
1563 1563 dtype_index = get_dtype_index(self.dtype)
1564 1564 procflag_dtype = get_procflag_dtype(dtype_index)
1565 1565
1566 1566 processFlags += procflag_dtype
1567 1567
1568 1568 if self.dataOut.flagDecodeData:
1569 1569 processFlags += PROCFLAG.DECODE_DATA
1570 1570
1571 1571 if self.dataOut.flagDeflipData:
1572 1572 processFlags += PROCFLAG.DEFLIP_DATA
1573 1573
1574 1574 if self.dataOut.code is not None:
1575 1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1576 1576
1577 1577 if self.dataOut.nCohInt > 1:
1578 1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1579 1579
1580 1580 if self.dataOut.type == "Spectra":
1581 1581 if self.dataOut.nIncohInt > 1:
1582 1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1583 1583
1584 1584 if self.dataOut.data_dc is not None:
1585 1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1586 1586
1587 1587 if self.dataOut.flagShiftFFT:
1588 1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1589 1589
1590 1590 return processFlags
1591 1591
1592 1592 def setBasicHeader(self):
1593 1593
1594 1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1595 1595 self.basicHeaderObj.version = self.versionFile
1596 1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1597 1597
1598 1598 utc = numpy.floor(self.dataOut.utctime)
1599 1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1600 1600
1601 1601 self.basicHeaderObj.utc = utc
1602 1602 self.basicHeaderObj.miliSecond = milisecond
1603 1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1604 1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1605 1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1606 1606
1607 1607 def setFirstHeader(self):
1608 1608 """
1609 1609 Obtiene una copia del First Header
1610 1610
1611 1611 Affected:
1612 1612
1613 1613 self.basicHeaderObj
1614 1614 self.systemHeaderObj
1615 1615 self.radarControllerHeaderObj
1616 1616 self.processingHeaderObj self.
1617 1617
1618 1618 Return:
1619 1619 None
1620 1620 """
1621 1621
1622 1622 raise NotImplementedError
1623 1623
1624 1624 def __writeFirstHeader(self):
1625 1625 """
1626 1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1627 1627
1628 1628 Affected:
1629 1629 __dataType
1630 1630
1631 1631 Return:
1632 1632 None
1633 1633 """
1634 1634
1635 1635 # CALCULAR PARAMETROS
1636 1636
1637 1637 sizeLongHeader = self.systemHeaderObj.size + \
1638 1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1639 1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1640 1640
1641 1641 self.basicHeaderObj.write(self.fp)
1642 1642 self.systemHeaderObj.write(self.fp)
1643 1643 self.radarControllerHeaderObj.write(self.fp)
1644 1644 self.processingHeaderObj.write(self.fp)
1645 1645
1646 1646 def __setNewBlock(self):
1647 1647 """
1648 1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1649 1649
1650 1650 Return:
1651 1651 0 : si no pudo escribir nada
1652 1652 1 : Si escribio el Basic el First Header
1653 1653 """
1654 1654 if self.fp == None:
1655 1655 self.setNextFile()
1656 1656
1657 1657 if self.flagIsNewFile:
1658 1658 return 1
1659 1659
1660 1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1661 1661 self.basicHeaderObj.write(self.fp)
1662 1662 return 1
1663 1663
1664 1664 if not(self.setNextFile()):
1665 1665 return 0
1666 1666
1667 1667 return 1
1668 1668
1669 1669 def writeNextBlock(self):
1670 1670 """
1671 1671 Selecciona el bloque siguiente de datos y los escribe en un file
1672 1672
1673 1673 Return:
1674 1674 0 : Si no hizo pudo escribir el bloque de datos
1675 1675 1 : Si no pudo escribir el bloque de datos
1676 1676 """
1677 1677 if not(self.__setNewBlock()):
1678 1678 return 0
1679 1679
1680 1680 self.writeBlock()
1681 1681
1682 1682 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1683 1683 self.processingHeaderObj.dataBlocksPerFile))
1684 1684
1685 1685 return 1
1686 1686
1687 1687 def setNextFile(self):
1688 1688 """
1689 1689 Determina el siguiente file que sera escrito
1690 1690
1691 1691 Affected:
1692 1692 self.filename
1693 1693 self.subfolder
1694 1694 self.fp
1695 1695 self.setFile
1696 1696 self.flagIsNewFile
1697 1697
1698 1698 Return:
1699 1699 0 : Si el archivo no puede ser escrito
1700 1700 1 : Si el archivo esta listo para ser escrito
1701 1701 """
1702 1702 ext = self.ext
1703 1703 path = self.path
1704 1704
1705 1705 if self.fp != None:
1706 1706 self.fp.close()
1707 1707
1708 1708 timeTuple = time.localtime(self.dataOut.utctime)
1709 1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1710 1710
1711 1711 fullpath = os.path.join(path, subfolder)
1712 1712 setFile = self.setFile
1713 1713
1714 1714 if not(os.path.exists(fullpath)):
1715 1715 os.mkdir(fullpath)
1716 1716 setFile = -1 # inicializo mi contador de seteo
1717 1717 else:
1718 1718 filesList = os.listdir(fullpath)
1719 1719 if len(filesList) > 0:
1720 1720 filesList = sorted(filesList, key=str.lower)
1721 1721 filen = filesList[-1]
1722 1722 # el filename debera tener el siguiente formato
1723 1723 # 0 1234 567 89A BCDE (hex)
1724 1724 # x YYYY DDD SSS .ext
1725 1725 if isNumber(filen[8:11]):
1726 1726 # inicializo mi contador de seteo al seteo del ultimo file
1727 1727 setFile = int(filen[8:11])
1728 1728 else:
1729 1729 setFile = -1
1730 1730 else:
1731 1731 setFile = -1 # inicializo mi contador de seteo
1732 1732
1733 1733 setFile += 1
1734 1734
1735 1735 # If this is a new day it resets some values
1736 1736 if self.dataOut.datatime.date() > self.fileDate:
1737 1737 setFile = 0
1738 1738 self.nTotalBlocks = 0
1739 1739
1740 1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1741 1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1742 1742
1743 1743 filename = os.path.join(path, subfolder, filen)
1744 1744
1745 1745 fp = open(filename, 'wb')
1746 1746
1747 1747 self.blockIndex = 0
1748 1748
1749 1749 # guardando atributos
1750 1750 self.filename = filename
1751 1751 self.subfolder = subfolder
1752 1752 self.fp = fp
1753 1753 self.setFile = setFile
1754 1754 self.flagIsNewFile = 1
1755 1755 self.fileDate = self.dataOut.datatime.date()
1756 1756
1757 1757 self.setFirstHeader()
1758 1758
1759 1759 print('[Writing] Opening file: %s' % self.filename)
1760 1760
1761 1761 self.__writeFirstHeader()
1762 1762
1763 1763 return 1
1764 1764
1765 1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1766 1766 """
1767 1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1768 1768
1769 1769 Inputs:
1770 1770 path : directory where data will be saved
1771 1771 profilesPerBlock : number of profiles per block
1772 1772 set : initial file set
1773 1773 datatype : An integer number that defines data type:
1774 1774 0 : int8 (1 byte)
1775 1775 1 : int16 (2 bytes)
1776 1776 2 : int32 (4 bytes)
1777 1777 3 : int64 (8 bytes)
1778 1778 4 : float32 (4 bytes)
1779 1779 5 : double64 (8 bytes)
1780 1780
1781 1781 Return:
1782 1782 0 : Si no realizo un buen seteo
1783 1783 1 : Si realizo un buen seteo
1784 1784 """
1785 1785
1786 1786 if ext == None:
1787 1787 ext = self.ext
1788 1788
1789 1789 self.ext = ext.lower()
1790 1790
1791 1791 self.path = path
1792 1792
1793 1793 if set is None:
1794 1794 self.setFile = -1
1795 1795 else:
1796 1796 self.setFile = set - 1
1797 1797
1798 1798 self.blocksPerFile = blocksPerFile
1799 1799
1800 1800 self.profilesPerBlock = profilesPerBlock
1801 1801
1802 1802 self.dataOut = dataOut
1803 1803 self.fileDate = self.dataOut.datatime.date()
1804 1804 # By default
1805 1805 self.dtype = self.dataOut.dtype
1806 1806
1807 1807 if datatype is not None:
1808 1808 self.dtype = get_numpy_dtype(datatype)
1809 1809
1810 1810 if not(self.setNextFile()):
1811 1811 print("[Writing] There isn't a next file")
1812 1812 return 0
1813 1813
1814 1814 self.setBlockDimension()
1815 1815
1816 1816 return 1
1817 1817
1818 1818 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1819 1819
1820 1820 if not(self.isConfig):
1821 1821
1822 1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1823 1823 set=set, ext=ext, datatype=datatype, **kwargs)
1824 1824 self.isConfig = True
1825 1825
1826 1826 self.putData() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,679 +1,680
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import numpy
7 7
8 8 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 10 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 11 from schainpy.model.data.jrodata import Spectra
12 12
13 @MPDecorator
13 14 class SpectraReader(JRODataReader, ProcessingUnit):
14 15 """
15 16 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
16 17 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
17 18 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
18 19
19 20 paresCanalesIguales * alturas * perfiles (Self Spectra)
20 21 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
21 22 canales * alturas (DC Channels)
22 23
23 24 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
24 25 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
25 26 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
26 27 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
27 28
28 29 Example:
29 30 dpath = "/home/myuser/data"
30 31
31 32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
32 33
33 34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
34 35
35 36 readerObj = SpectraReader()
36 37
37 38 readerObj.setup(dpath, startTime, endTime)
38 39
39 40 while(True):
40 41
41 42 readerObj.getData()
42 43
43 44 print readerObj.data_spc
44 45
45 46 print readerObj.data_cspc
46 47
47 48 print readerObj.data_dc
48 49
49 50 if readerObj.flagNoMoreFiles:
50 51 break
51 52
52 53 """
53 54
54 55 pts2read_SelfSpectra = 0
55 56
56 57 pts2read_CrossSpectra = 0
57 58
58 59 pts2read_DCchannels = 0
59 60
60 61 ext = ".pdata"
61 62
62 63 optchar = "P"
63 64
64 65 dataOut = None
65 66
66 67 nRdChannels = None
67 68
68 69 nRdPairs = None
69 70
70 71 rdPairList = []
71 72
72 def __init__(self, **kwargs):
73 def __init__(self):#, **kwargs):
73 74 """
74 75 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
75 76
76 77 Inputs:
77 78 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
78 79 almacenar un perfil de datos cada vez que se haga un requerimiento
79 80 (getData). El perfil sera obtenido a partir del buffer de datos,
80 81 si el buffer esta vacio se hara un nuevo proceso de lectura de un
81 82 bloque de datos.
82 83 Si este parametro no es pasado se creara uno internamente.
83 84
84 85 Affected:
85 86 self.dataOut
86 87
87 88 Return : None
88 89 """
89 90
90 91 #Eliminar de la base la herencia
91 ProcessingUnit.__init__(self, **kwargs)
92 ProcessingUnit.__init__(self)#, **kwargs)
92 93
93 94 # self.isConfig = False
94 95
95 96 self.pts2read_SelfSpectra = 0
96 97
97 98 self.pts2read_CrossSpectra = 0
98 99
99 100 self.pts2read_DCchannels = 0
100 101
101 102 self.datablock = None
102 103
103 104 self.utc = None
104 105
105 106 self.ext = ".pdata"
106 107
107 108 self.optchar = "P"
108 109
109 110 self.basicHeaderObj = BasicHeader(LOCALTIME)
110 111
111 112 self.systemHeaderObj = SystemHeader()
112 113
113 114 self.radarControllerHeaderObj = RadarControllerHeader()
114 115
115 116 self.processingHeaderObj = ProcessingHeader()
116 117
117 118 self.online = 0
118 119
119 120 self.fp = None
120 121
121 122 self.idFile = None
122 123
123 124 self.dtype = None
124 125
125 126 self.fileSizeByHeader = None
126 127
127 128 self.filenameList = []
128 129
129 130 self.filename = None
130 131
131 132 self.fileSize = None
132 133
133 134 self.firstHeaderSize = 0
134 135
135 136 self.basicHeaderSize = 24
136 137
137 138 self.pathList = []
138 139
139 140 self.lastUTTime = 0
140 141
141 142 self.maxTimeStep = 30
142 143
143 144 self.flagNoMoreFiles = 0
144 145
145 146 self.set = 0
146 147
147 148 self.path = None
148 149
149 150 self.delay = 60 #seconds
150 151
151 152 self.nTries = 3 #quantity tries
152 153
153 154 self.nFiles = 3 #number of files for searching
154 155
155 156 self.nReadBlocks = 0
156 157
157 158 self.flagIsNewFile = 1
158 159
159 160 self.__isFirstTimeOnline = 1
160 161
161 162 # self.ippSeconds = 0
162 163
163 164 self.flagDiscontinuousBlock = 0
164 165
165 166 self.flagIsNewBlock = 0
166 167
167 168 self.nTotalBlocks = 0
168 169
169 170 self.blocksize = 0
170 171
171 172 self.dataOut = self.createObjByDefault()
172 173
173 174 self.profileIndex = 1 #Always
174 175
175 176
176 177 def createObjByDefault(self):
177 178
178 179 dataObj = Spectra()
179 180
180 181 return dataObj
181 182
182 183 def __hasNotDataInBuffer(self):
183 184 return 1
184 185
185 186
186 187 def getBlockDimension(self):
187 188 """
188 189 Obtiene la cantidad de puntos a leer por cada bloque de datos
189 190
190 191 Affected:
191 192 self.nRdChannels
192 193 self.nRdPairs
193 194 self.pts2read_SelfSpectra
194 195 self.pts2read_CrossSpectra
195 196 self.pts2read_DCchannels
196 197 self.blocksize
197 198 self.dataOut.nChannels
198 199 self.dataOut.nPairs
199 200
200 201 Return:
201 202 None
202 203 """
203 204 self.nRdChannels = 0
204 205 self.nRdPairs = 0
205 206 self.rdPairList = []
206 207
207 208 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
208 209 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
209 210 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
210 211 else:
211 212 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
212 213 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
213 214
214 215 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
215 216
216 217 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
217 218 self.blocksize = self.pts2read_SelfSpectra
218 219
219 220 if self.processingHeaderObj.flag_cspc:
220 221 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
221 222 self.blocksize += self.pts2read_CrossSpectra
222 223
223 224 if self.processingHeaderObj.flag_dc:
224 225 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
225 226 self.blocksize += self.pts2read_DCchannels
226 227
227 228 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
228 229
229 230
230 231 def readBlock(self):
231 232 """
232 233 Lee el bloque de datos desde la posicion actual del puntero del archivo
233 234 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
234 235 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
235 236 es seteado a 0
236 237
237 238 Return: None
238 239
239 240 Variables afectadas:
240 241
241 242 self.flagIsNewFile
242 243 self.flagIsNewBlock
243 244 self.nTotalBlocks
244 245 self.data_spc
245 246 self.data_cspc
246 247 self.data_dc
247 248
248 249 Exceptions:
249 250 Si un bloque leido no es un bloque valido
250 251 """
251 252 blockOk_flag = False
252 253 fpointer = self.fp.tell()
253 254
254 255 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
255 256 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
256 257
257 258 if self.processingHeaderObj.flag_cspc:
258 259 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
259 260 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
260 261
261 262 if self.processingHeaderObj.flag_dc:
262 263 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
263 264 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
264 265
265 266
266 267 if self.processingHeaderObj.shif_fft:
267 268 #desplaza a la derecha en el eje 2 determinadas posiciones
268 269 shift = int(self.processingHeaderObj.profilesPerBlock/2)
269 270 spc = numpy.roll( spc, shift , axis=2 )
270 271
271 272 if self.processingHeaderObj.flag_cspc:
272 273 #desplaza a la derecha en el eje 2 determinadas posiciones
273 274 cspc = numpy.roll( cspc, shift, axis=2 )
274 275
275 276 #Dimensions : nChannels, nProfiles, nSamples
276 277 spc = numpy.transpose( spc, (0,2,1) )
277 278 self.data_spc = spc
278 279
279 280 if self.processingHeaderObj.flag_cspc:
280 281 cspc = numpy.transpose( cspc, (0,2,1) )
281 282 self.data_cspc = cspc['real'] + cspc['imag']*1j
282 283 else:
283 284 self.data_cspc = None
284 285
285 286 if self.processingHeaderObj.flag_dc:
286 287 self.data_dc = dc['real'] + dc['imag']*1j
287 288 else:
288 289 self.data_dc = None
289 290
290 291 self.flagIsNewFile = 0
291 292 self.flagIsNewBlock = 1
292 293
293 294 self.nTotalBlocks += 1
294 295 self.nReadBlocks += 1
295 296
296 297 return 1
297 298
298 299 def getFirstHeader(self):
299 300
300 301 self.getBasicHeader()
301 302
302 303 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
303 304
304 305 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
305 306
306 307 # self.dataOut.ippSeconds = self.ippSeconds
307 308
308 309 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
309 310
310 311 self.dataOut.dtype = self.dtype
311 312
312 313 # self.dataOut.nPairs = self.nPairs
313 314
314 315 self.dataOut.pairsList = self.rdPairList
315 316
316 317 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
317 318
318 319 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
319 320
320 321 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
321 322
322 323 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
323 324
324 325 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
325 326
326 327 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
327 328
328 329 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
329 330
330 331 self.dataOut.flagShiftFFT = True #Data is always shifted
331 332
332 333 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
333 334
334 335 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
335 336
336 337 def getData(self):
337 338 """
338 339 First method to execute before "RUN" is called.
339 340
340 341 Copia el buffer de lectura a la clase "Spectra",
341 342 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
342 343 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
343 344
344 345 Return:
345 346 0 : Si no hay mas archivos disponibles
346 347 1 : Si hizo una buena copia del buffer
347 348
348 349 Affected:
349 350 self.dataOut
350 351
351 352 self.flagDiscontinuousBlock
352 353 self.flagIsNewBlock
353 354 """
354 355
355 356 if self.flagNoMoreFiles:
356 357 self.dataOut.flagNoData = True
357 358 print('Process finished')
358 359 return 0
359 360
360 361 self.flagDiscontinuousBlock = 0
361 362 self.flagIsNewBlock = 0
362 363
363 364 if self.__hasNotDataInBuffer():
364 365
365 366 if not( self.readNextBlock() ):
366 367 self.dataOut.flagNoData = True
367 368 return 0
368 369
369 370 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
370 371
371 372 if self.data_spc is None:
372 373 self.dataOut.flagNoData = True
373 374 return 0
374 375
375 376 self.getBasicHeader()
376 377
377 378 self.getFirstHeader()
378 379
379 380 self.dataOut.data_spc = self.data_spc
380 381
381 382 self.dataOut.data_cspc = self.data_cspc
382 383
383 384 self.dataOut.data_dc = self.data_dc
384 385
385 386 self.dataOut.flagNoData = False
386 387
387 388 self.dataOut.realtime = self.online
388 389
389 390 return self.dataOut.data_spc
390 391
391 392 class SpectraWriter(JRODataWriter, Operation):
392 393
393 394 """
394 395 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
395 396 de los datos siempre se realiza por bloques.
396 397 """
397 398
398 399 ext = ".pdata"
399 400
400 401 optchar = "P"
401 402
402 403 shape_spc_Buffer = None
403 404
404 405 shape_cspc_Buffer = None
405 406
406 407 shape_dc_Buffer = None
407 408
408 409 data_spc = None
409 410
410 411 data_cspc = None
411 412
412 413 data_dc = None
413 414
414 415 # dataOut = None
415 416
416 417 def __init__(self, **kwargs):
417 418 """
418 419 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
419 420
420 421 Affected:
421 422 self.dataOut
422 423 self.basicHeaderObj
423 424 self.systemHeaderObj
424 425 self.radarControllerHeaderObj
425 426 self.processingHeaderObj
426 427
427 428 Return: None
428 429 """
429 430
430 431 Operation.__init__(self, **kwargs)
431 432
432 433 self.isConfig = False
433 434
434 435 self.nTotalBlocks = 0
435 436
436 437 self.data_spc = None
437 438
438 439 self.data_cspc = None
439 440
440 441 self.data_dc = None
441 442
442 443 self.fp = None
443 444
444 445 self.flagIsNewFile = 1
445 446
446 447 self.nTotalBlocks = 0
447 448
448 449 self.flagIsNewBlock = 0
449 450
450 451 self.setFile = None
451 452
452 453 self.dtype = None
453 454
454 455 self.path = None
455 456
456 457 self.noMoreFiles = 0
457 458
458 459 self.filename = None
459 460
460 461 self.basicHeaderObj = BasicHeader(LOCALTIME)
461 462
462 463 self.systemHeaderObj = SystemHeader()
463 464
464 465 self.radarControllerHeaderObj = RadarControllerHeader()
465 466
466 467 self.processingHeaderObj = ProcessingHeader()
467 468
468 469
469 470 def hasAllDataInBuffer(self):
470 471 return 1
471 472
472 473
473 474 def setBlockDimension(self):
474 475 """
475 476 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
476 477
477 478 Affected:
478 479 self.shape_spc_Buffer
479 480 self.shape_cspc_Buffer
480 481 self.shape_dc_Buffer
481 482
482 483 Return: None
483 484 """
484 485 self.shape_spc_Buffer = (self.dataOut.nChannels,
485 486 self.processingHeaderObj.nHeights,
486 487 self.processingHeaderObj.profilesPerBlock)
487 488
488 489 self.shape_cspc_Buffer = (self.dataOut.nPairs,
489 490 self.processingHeaderObj.nHeights,
490 491 self.processingHeaderObj.profilesPerBlock)
491 492
492 493 self.shape_dc_Buffer = (self.dataOut.nChannels,
493 494 self.processingHeaderObj.nHeights)
494 495
495 496
496 497 def writeBlock(self):
497 498 """
498 499 Escribe el buffer en el file designado
499 500
500 501 Affected:
501 502 self.data_spc
502 503 self.data_cspc
503 504 self.data_dc
504 505 self.flagIsNewFile
505 506 self.flagIsNewBlock
506 507 self.nTotalBlocks
507 508 self.nWriteBlocks
508 509
509 510 Return: None
510 511 """
511 512
512 513 spc = numpy.transpose( self.data_spc, (0,2,1) )
513 if self.processingHeaderObj.shif_fft:
514 if not( self.processingHeaderObj.shif_fft ):
514 515 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
515 516 data = spc.reshape((-1))
516 517 data = data.astype(self.dtype[0])
517 518 data.tofile(self.fp)
518 519
519 520 if self.data_cspc is not None:
520 521 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
521 522 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
522 if self.processingHeaderObj.shif_fft:
523 if not( self.processingHeaderObj.shif_fft ):
523 524 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
524 525 data['real'] = cspc.real
525 526 data['imag'] = cspc.imag
526 527 data = data.reshape((-1))
527 528 data.tofile(self.fp)
528 529
529 530 if self.data_dc is not None:
530 531 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
531 532 dc = self.data_dc
532 533 data['real'] = dc.real
533 534 data['imag'] = dc.imag
534 535 data = data.reshape((-1))
535 536 data.tofile(self.fp)
536 537
537 538 # self.data_spc.fill(0)
538 539 #
539 540 # if self.data_dc is not None:
540 541 # self.data_dc.fill(0)
541 542 #
542 543 # if self.data_cspc is not None:
543 544 # self.data_cspc.fill(0)
544 545
545 546 self.flagIsNewFile = 0
546 547 self.flagIsNewBlock = 1
547 548 self.nTotalBlocks += 1
548 549 self.nWriteBlocks += 1
549 550 self.blockIndex += 1
550 551
551 552 # print "[Writing] Block = %d04" %self.blockIndex
552 553
553 554 def putData(self):
554 555 """
555 556 Setea un bloque de datos y luego los escribe en un file
556 557
557 558 Affected:
558 559 self.data_spc
559 560 self.data_cspc
560 561 self.data_dc
561 562
562 563 Return:
563 564 0 : Si no hay data o no hay mas files que puedan escribirse
564 565 1 : Si se escribio la data de un bloque en un file
565 566 """
566 567
567 568 if self.dataOut.flagNoData:
568 569 return 0
569 570
570 571 self.flagIsNewBlock = 0
571 572
572 573 if self.dataOut.flagDiscontinuousBlock:
573 574 self.data_spc.fill(0)
574 575 if self.dataOut.data_cspc is not None:
575 576 self.data_cspc.fill(0)
576 577 if self.dataOut.data_dc is not None:
577 578 self.data_dc.fill(0)
578 579 self.setNextFile()
579 580
580 581 if self.flagIsNewFile == 0:
581 582 self.setBasicHeader()
582 583
583 584 self.data_spc = self.dataOut.data_spc.copy()
584 585
585 586 if self.dataOut.data_cspc is not None:
586 587 self.data_cspc = self.dataOut.data_cspc.copy()
587 588
588 589 if self.dataOut.data_dc is not None:
589 590 self.data_dc = self.dataOut.data_dc.copy()
590 591
591 592 # #self.processingHeaderObj.dataBlocksPerFile)
592 593 if self.hasAllDataInBuffer():
593 594 # self.setFirstHeader()
594 595 self.writeNextBlock()
595 596
596 597 return 1
597 598
598 599 def __getBlockSize(self):
599 600 '''
600 601 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
601 602 '''
602 603
603 604 dtype_width = self.getDtypeWidth()
604 605
605 606 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
606 607
607 608 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
608 609 blocksize = (pts2write_SelfSpectra*dtype_width)
609 610
610 611 if self.dataOut.data_cspc is not None:
611 612 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
612 613 blocksize += (pts2write_CrossSpectra*dtype_width*2)
613 614
614 615 if self.dataOut.data_dc is not None:
615 616 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
616 617 blocksize += (pts2write_DCchannels*dtype_width*2)
617 618
618 619 # blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
619 620
620 621 return blocksize
621 622
622 623 def setFirstHeader(self):
623 624
624 625 """
625 626 Obtiene una copia del First Header
626 627
627 628 Affected:
628 629 self.systemHeaderObj
629 630 self.radarControllerHeaderObj
630 631 self.dtype
631 632
632 633 Return:
633 634 None
634 635 """
635 636
636 637 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
637 638 self.systemHeaderObj.nChannels = self.dataOut.nChannels
638 639 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
639 640
640 641 self.processingHeaderObj.dtype = 1 # Spectra
641 642 self.processingHeaderObj.blockSize = self.__getBlockSize()
642 643 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
643 644 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
644 645 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
645 646 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
646 647 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
647 648 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
648 649 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
649 650
650 651 if self.processingHeaderObj.totalSpectra > 0:
651 652 channelList = []
652 653 for channel in range(self.dataOut.nChannels):
653 654 channelList.append(channel)
654 655 channelList.append(channel)
655 656
656 657 pairsList = []
657 658 if self.dataOut.nPairs > 0:
658 659 for pair in self.dataOut.pairsList:
659 660 pairsList.append(pair[0])
660 661 pairsList.append(pair[1])
661 662
662 663 spectraComb = channelList + pairsList
663 664 spectraComb = numpy.array(spectraComb, dtype="u1")
664 665 self.processingHeaderObj.spectraComb = spectraComb
665 666
666 667 if self.dataOut.code is not None:
667 668 self.processingHeaderObj.code = self.dataOut.code
668 669 self.processingHeaderObj.nCode = self.dataOut.nCode
669 670 self.processingHeaderObj.nBaud = self.dataOut.nBaud
670 671
671 672 if self.processingHeaderObj.nWindows != 0:
672 673 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
673 674 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
674 675 self.processingHeaderObj.nHeights = self.dataOut.nHeights
675 676 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
676 677
677 678 self.processingHeaderObj.processFlags = self.getProcessFlags()
678 679
679 680 self.setBasicHeader() No newline at end of file
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (606 lines changed) Show them Hide them
@@ -1,360 +1,538
1 1 '''
2
3 $Author: murco $
4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
2 Updated for multiprocessing
3 Author : Sergio Cortez
4 Jan 2018
5 Abstract:
6 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
7 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
8 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
9
10 Based on:
11 $Author: murco $
12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
5 13 '''
14 from platform import python_version
6 15 import inspect
7 from fuzzywuzzy import process
8
9 def checkKwargs(method, kwargs):
10 currentKwargs = kwargs
11 choices = inspect.getargspec(method).args
12 try:
13 choices.remove('self')
14 except Exception as e:
15 pass
16 import zmq
17 import time
18 import pickle
19 import os
20 from multiprocessing import Process
16 21
17 try:
18 choices.remove('dataOut')
19 except Exception as e:
20 pass
22 from schainpy.utils import log
21 23
22 for kwarg in kwargs:
23 fuzz = process.extractOne(kwarg, choices)
24 if fuzz is None:
25 continue
26 if fuzz[1] < 100:
27 raise Exception('\x1b[0;32;40mDid you mean {} instead of {} in {}? \x1b[0m'.
28 format(fuzz[0], kwarg, method.__self__.__class__.__name__))
29 24
30 25 class ProcessingUnit(object):
31 26
32 27 """
33 Esta es la clase base para el procesamiento de datos.
28 Update - Jan 2018 - MULTIPROCESSING
29 All the "call" methods present in the previous base were removed.
30 The majority of operations are independant processes, thus
31 the decorator is in charge of communicate the operation processes
32 with the proccessing unit via IPC.
34 33
35 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
36 - Metodos internos (callMethod)
37 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
38 tienen que ser agreagados con el metodo "add".
34 The constructor does not receive any argument. The remaining methods
35 are related with the operations to execute.
36
39 37
40 38 """
41 39 # objeto de datos de entrada (Voltage, Spectra o Correlation)
42 40 dataIn = None
43 41 dataInList = []
44 42
45 43 # objeto de datos de entrada (Voltage, Spectra o Correlation)
44
45 id = None
46 inputId = None
47
46 48 dataOut = None
47 49
50 dictProcs = None
51
48 52 operations2RunDict = None
49 53
50 54 isConfig = False
51 55
52
53 def __init__(self, *args, **kwargs):
56 def __init__(self):
54 57
55 58 self.dataIn = None
56 self.dataInList = []
57
58 59 self.dataOut = None
59 60
60 self.operations2RunDict = {}
61 self.operationKwargs = {}
62
63 61 self.isConfig = False
64 62
65 self.args = args
66 self.kwargs = kwargs
67
68 if not hasattr(self, 'name'):
69 self.name = self.__class__.__name__
70
71 checkKwargs(self.run, kwargs)
72
73 63 def getAllowedArgs(self):
74 64 if hasattr(self, '__attrs__'):
75 65 return self.__attrs__
76 66 else:
77 67 return inspect.getargspec(self.run).args
78 68
79 69 def addOperationKwargs(self, objId, **kwargs):
80 70 '''
81 71 '''
82 72
83 73 self.operationKwargs[objId] = kwargs
84
85
74
86 75 def addOperation(self, opObj, objId):
87 76
88 77 """
89 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
90 identificador asociado a este objeto.
78 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
79 posses the id of the operation process (IPC purposes)
91 80
92 Input:
81 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
82 identificador asociado a este objeto.
93 83
94 object : objeto de la clase "Operation"
84 Input:
95 85
96 Return:
86 object : objeto de la clase "Operation"
87
88 Return:
97 89
98 objId : identificador del objeto, necesario para ejecutar la operacion
90 objId : identificador del objeto, necesario para comunicar con master(procUnit)
99 91 """
100 92
101 93 self.operations2RunDict[objId] = opObj
102 94
103 95 return objId
104 96
97
105 98 def getOperationObj(self, objId):
106 99
107 100 if objId not in list(self.operations2RunDict.keys()):
108 101 return None
109 102
110 103 return self.operations2RunDict[objId]
111 104
112 105 def operation(self, **kwargs):
113 106
114 107 """
115 108 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
116 109 atributos del objeto dataOut
117 110
118 111 Input:
119 112
120 113 **kwargs : Diccionario de argumentos de la funcion a ejecutar
121 114 """
122 115
123 raise NotImplementedError
124
125 def callMethod(self, name, opId):
126
127 """
128 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
129
130 Input:
131 name : nombre del metodo a ejecutar
132
133 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
134
135 """
116 raise NotImplementedError
136 117
137 #Checking the inputs
138 if name == 'run':
139
140 if not self.checkInputs():
141 self.dataOut.flagNoData = True
142 return False
143 else:
144 #Si no es un metodo RUN la entrada es la misma dataOut (interna)
145 if self.dataOut is not None and self.dataOut.isEmpty():
146 return False
118 def setup(self):
147 119
148 #Getting the pointer to method
149 methodToCall = getattr(self, name)
120 raise NotImplementedError
150 121
151 #Executing the self method
122 def run(self):
152 123
153 if hasattr(self, 'mp'):
154 if name=='run':
155 if self.mp is False:
156 self.mp = True
157 self.start()
158 else:
159 self.operationKwargs[opId]['parent'] = self.kwargs
160 methodToCall(**self.operationKwargs[opId])
161 else:
162 if name=='run':
163 methodToCall(**self.kwargs)
164 else:
165 methodToCall(**self.operationKwargs[opId])
124 raise NotImplementedError
166 125
167 if self.dataOut is None:
168 return False
126 def close(self):
127 #Close every thread, queue or any other object here is it is neccesary.
128 return
129
130 class Operation(object):
169 131
170 if self.dataOut.isEmpty():
171 return False
132 """
133 Update - Jan 2018 - MULTIPROCESSING
172 134
173 return True
135 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
136 The constructor doe snot receive any argument, neither the baseclass.
174 137
175 def callObject(self, objId):
176 138
177 """
178 Ejecuta la operacion asociada al identificador del objeto "objId"
139 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
140 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
141 acumulacion dentro de esta clase
179 142
180 Input:
143 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
181 144
182 objId : identificador del objeto a ejecutar
145 """
146 id = None
147 __buffer = None
148 dest = None
149 isConfig = False
150 readyFlag = None
183 151
184 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
152 def __init__(self):
185 153
186 Return:
154 self.buffer = None
155 self.dest = None
156 self.isConfig = False
157 self.readyFlag = False
187 158
188 None
189 """
159 if not hasattr(self, 'name'):
160 self.name = self.__class__.__name__
161
162 def getAllowedArgs(self):
163 if hasattr(self, '__attrs__'):
164 return self.__attrs__
165 else:
166 return inspect.getargspec(self.run).args
190 167
191 if self.dataOut is not None and self.dataOut.isEmpty():
192 return False
168 def setup(self):
193 169
194 externalProcObj = self.operations2RunDict[objId]
170 self.isConfig = True
195 171
196 if hasattr(externalProcObj, 'mp'):
197 if externalProcObj.mp is False:
198 externalProcObj.kwargs['parent'] = self.kwargs
199 self.operationKwargs[objId] = externalProcObj.kwargs
200 externalProcObj.mp = True
201 externalProcObj.start()
202 else:
203 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
204 self.operationKwargs[objId] = externalProcObj.kwargs
172 raise NotImplementedError
205 173
206 174
207 return True
175 def run(self, dataIn, **kwargs):
208 176
209 def call(self, opType, opName=None, opId=None):
210 177 """
211 Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
212 identificada con el id "opId"; con los argumentos "**kwargs".
213
214 False si la operacion no se ha ejecutado.
178 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
179 atributos del objeto dataIn.
215 180
216 181 Input:
217 182
218 opType : Puede ser "self" o "external"
183 dataIn : objeto del tipo JROData
219 184
220 Depende del tipo de operacion para llamar a:callMethod or callObject:
185 Return:
221 186
222 1. If opType = "self": Llama a un metodo propio de esta clase:
187 None
223 188
224 name_method = getattr(self, name)
225 name_method(**kwargs)
189 Affected:
190 __buffer : buffer de recepcion de datos.
226 191
192 """
193 if not self.isConfig:
194 self.setup(**kwargs)
227 195
228 2. If opType = "other" o"external": Llama al metodo "run()" de una instancia de la
229 clase "Operation" o de un derivado de ella:
196 raise NotImplementedError
230 197
231 instanceName = self.operationList[opId]
232 instanceName.run(**kwargs)
198 def close(self):
233 199
234 opName : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
235 usada para llamar a un metodo interno de la clase Processing
200 pass
236 201
237 opId : Si la operacion es externa (opType = 'other' o 'external), entonces el
238 "opId" sera usada para llamar al metodo "run" de la clase Operation
239 registrada anteriormente con ese Id
240 202
241 Exception:
242 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
243 "addOperation" e identificado con el valor "opId" = el id de la operacion.
244 De lo contrario retornara un error del tipo ValueError
203 ######### Decorator #########
245 204
246 """
247 205
248 if opType == 'self':
206 def MPDecorator(BaseClass):
207
208 """
209 "Multiprocessing class decorator"
249 210
250 if not opName:
251 raise ValueError("opName parameter should be defined")
211 This function add multiprocessing features to the base class. Also,
212 it handle the communication beetween processes (readers, procUnits and operations).
213 Receive the arguments at the moment of instantiation. According to that, discriminates if it
214 is a procUnit or an operation
215 """
216
217 class MPClass(BaseClass, Process):
218
219 "This is the overwritten class"
220 operations2RunDict = None
221 socket_l = None
222 socket_p = None
223 socketOP = None
224 socket_router = None
225 dictProcs = None
226 typeProc = None
227 def __init__(self, *args, **kwargs):
228 super(MPClass, self).__init__()
229 Process.__init__(self)
230
231
232 self.operationKwargs = {}
233 self.args = args
234
235
236 self.operations2RunDict = {}
237 self.kwargs = kwargs
238
239 # The number of arguments (args) determine the type of process
240
241 if len(self.args) is 3:
242 self.typeProc = "ProcUnit"
243 self.id = args[0] #topico de publicacion
244 self.inputId = args[1] #topico de subcripcion
245 self.dictProcs = args[2] #diccionario de procesos globales
246 else:
247 self.id = args[0]
248 self.typeProc = "Operation"
249
250 def addOperationKwargs(self, objId, **kwargs):
251
252 self.operationKwargs[objId] = kwargs
252 253
253 sts = self.callMethod(opName, opId)
254 def getAllowedArgs(self):
254 255
255 elif opType == 'other' or opType == 'external' or opType == 'plotter':
256 if hasattr(self, '__attrs__'):
257 return self.__attrs__
258 else:
259 return inspect.getargspec(self.run).args
260
261
262 def sockListening(self, topic):
263
264 """
265 This function create a socket to receive objects.
266 The 'topic' argument is related to the publisher process from which the self process is
267 listening (data).
268 In the case were the self process is listening to a Reader (proc Unit),
269 special conditions are introduced to maximize parallelism.
270 """
271
272 cont = zmq.Context()
273 zmq_socket = cont.socket(zmq.SUB)
274 if not os.path.exists('/tmp/socketTmp'):
275 os.mkdir('/tmp/socketTmp')
276
277 if 'Reader' in self.dictProcs[self.inputId].name:
278 zmq_socket.connect('ipc:///tmp/socketTmp/b')
279
280 else:
281 zmq_socket.connect('ipc:///tmp/socketTmp/%s' % self.inputId)
282
283 #log.error('RECEIVING FROM {} {}'.format(self.inputId, str(topic).encode()))
284 zmq_socket.setsockopt(zmq.SUBSCRIBE, str(topic).encode()) #yong
256 285
257 if not opId:
258 raise ValueError("opId parameter should be defined")
286 return zmq_socket
259 287
260 if opId not in list(self.operations2RunDict.keys()):
261 raise ValueError("Any operation with id=%s has been added" %str(opId))
262 288
263 sts = self.callObject(opId)
289 def listenProc(self, sock):
264 290
265 else:
266 raise ValueError("opType should be 'self', 'external' or 'plotter'; and not '%s'" %opType)
291 """
292 This function listen to a ipc addres until a message is recovered. To serialize the
293 data (object), pickle has been use.
294 The 'sock' argument is the socket previously connect to an ipc address and with a topic subscription.
295 """
296
297 a = sock.recv_multipart()
298 a = pickle.loads(a[1])
299 return a
267 300
268 return sts
301 def sockPublishing(self):
269 302
270 def setInput(self, dataIn):
303 """
304 This function create a socket for publishing purposes.
305 Depending on the process type from where is created, it binds or connect
306 to special IPC addresses.
307 """
308 time.sleep(4) #yong
309 context = zmq.Context()
310 zmq_socket = context.socket(zmq.PUB)
311 if not os.path.exists('/tmp/socketTmp'): os.mkdir('/tmp/socketTmp')
312 if 'Reader' in self.dictProcs[self.id].name:
313 zmq_socket.connect('ipc:///tmp/socketTmp/a')
314 else:
315 zmq_socket.bind('ipc:///tmp/socketTmp/%s' % self.id)
271 316
272 self.dataIn = dataIn
273 self.dataInList.append(dataIn)
317 return zmq_socket
274 318
275 def getOutputObj(self):
319 def publishProc(self, sock, data):
276 320
277 return self.dataOut
321 """
322 This function publish a python object (data) under a specific topic in a socket (sock).
323 Usually, the topic is the self id of the process.
324 """
278 325
279 def checkInputs(self):
326 sock.send_multipart([str(self.id).encode(), pickle.dumps(data)]) #yong
327
328 return True
280 329
281 for thisDataIn in self.dataInList:
330 def sockOp(self):
282 331
283 if thisDataIn.isEmpty():
284 return False
332 """
333 This function create a socket for communication purposes with operation processes.
334 """
285 335
286 return True
336 cont = zmq.Context()
337 zmq_socket = cont.socket(zmq.DEALER)
338
339 if python_version()[0] == '2':
340 zmq_socket.setsockopt(zmq.IDENTITY, self.id)
341 if python_version()[0] == '3':
342 zmq_socket.setsockopt_string(zmq.IDENTITY, self.id)
287 343
288 def setup(self):
289 344
290 raise NotImplementedError
345 return zmq_socket
291 346
292 def run(self):
293 347
294 raise NotImplementedError
348 def execOp(self, socket, opId, dataObj):
295 349
296 def close(self):
297 #Close every thread, queue or any other object here is it is neccesary.
298 return
350 """
351 This function 'execute' an operation main routine by establishing a
352 connection with it and sending a python object (dataOut).
353 """
354 if not os.path.exists('/tmp/socketTmp'): os.mkdir('/tmp/socketTmp')
355 socket.connect('ipc:///tmp/socketTmp/%s' %opId)
356
357
358 socket.send(pickle.dumps(dataObj)) #yong
359
360 argument = socket.recv_multipart()[0]
361
362 argument = pickle.loads(argument)
363
364 return argument
365
366 def sockIO(self):
299 367
300 class Operation(object):
368 """
369 Socket defined for an operation process. It is able to recover the object sent from another process as well as a
370 identifier of who sent it.
371 """
301 372
302 """
303 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
304 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
305 acumulacion dentro de esta clase
373 cont = zmq.Context()
374 if not os.path.exists('/tmp/socketTmp'): os.mkdir('/tmp/socketTmp')
375 socket = cont.socket(zmq.ROUTER)
376 socket.bind('ipc:///tmp/socketTmp/%s' % self.id)
306 377
307 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
378 return socket
308 379
309 """
380 def funIOrec(self, socket):
310 381
311 __buffer = None
312 isConfig = False
382 """
383 Operation method, recover the id of the process who sent a python object.
384 The 'socket' argument is the socket binded to a specific process ipc.
385 """
313 386
314 def __init__(self, **kwargs):
387 #id_proc = socket.recv()
388
389 #dataObj = socket.recv_pyobj()
390
391 dataObj = socket.recv_multipart()
392
393 dataObj[1] = pickle.loads(dataObj[1])
394 return dataObj[0], dataObj[1]
395
396 def funIOsen(self, socket, data, dest):
397
398 """
399 Operation method, send a python object to a specific destination.
400 The 'dest' argument is the id of a proccesinf unit.
401 """
402
403 socket.send_multipart([dest, pickle.dumps(data)]) #yong
315 404
316 self.__buffer = None
317 self.isConfig = False
318 self.kwargs = kwargs
319 if not hasattr(self, 'name'):
320 self.name = self.__class__.__name__
321 checkKwargs(self.run, kwargs)
405 return True
322 406
323 def getAllowedArgs(self):
324 if hasattr(self, '__attrs__'):
325 return self.__attrs__
326 else:
327 return inspect.getargspec(self.run).args
328 407
329 def setup(self):
408 def runReader(self):
330 409
331 self.isConfig = True
410 # time.sleep(3)
411 while True:
412
413 BaseClass.run(self, **self.kwargs)
332 414
333 raise NotImplementedError
334 415
335 def run(self, dataIn, **kwargs):
416 keyList = list(self.operations2RunDict.keys())
417 keyList.sort()
418
419 for key in keyList:
420 self.socketOP = self.sockOp()
421 self.dataOut = self.execOp(self.socketOP, key, self.dataOut)
336 422
337 """
338 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
339 atributos del objeto dataIn.
423
424 if self.flagNoMoreFiles: #Usar un objeto con flags para saber si termino el proc o hubo un error
425 self.publishProc(self.socket_p, "Finish")
426 break
340 427
341 Input:
342
343 dataIn : objeto del tipo JROData
428 if self.dataOut.flagNoData:
429 continue
430
431 print("Publishing data...")
432 self.publishProc(self.socket_p, self.dataOut)
433 # time.sleep(2)
434
435
436 print("%s done" %BaseClass.__name__)
437 return 0
438
439 def runProc(self):
344 440
345 Return:
441 # All the procUnits with kwargs that require a setup initialization must be defined here.
346 442
347 None
443 if self.setupReq:
444 BaseClass.setup(self, **self.kwargs)
348 445
349 Affected:
350 __buffer : buffer de recepcion de datos.
446 while True:
447 self.dataIn = self.listenProc(self.socket_l)
448 print("%s received data" %BaseClass.__name__)
449
450 if self.dataIn == "Finish":
451 break
452
453 m_arg = list(self.kwargs.keys())
454 num_arg = list(range(1,int(BaseClass.run.__code__.co_argcount)))
455
456 run_arg = {}
457
458 for var in num_arg:
459 if BaseClass.run.__code__.co_varnames[var] in m_arg:
460 run_arg[BaseClass.run.__code__.co_varnames[var]] = self.kwargs[BaseClass.run.__code__.co_varnames[var]]
461
462 #BaseClass.run(self, **self.kwargs)
463 BaseClass.run(self, **run_arg)
464
465 ## Iterar sobre una serie de data que podrias aplicarse
466
467 for m_name in BaseClass.METHODS:
468
469 met_arg = {}
470
471 for arg in m_arg:
472 if arg in BaseClass.METHODS[m_name]:
473 for att in BaseClass.METHODS[m_name]:
474 met_arg[att] = self.kwargs[att]
475
476 method = getattr(BaseClass, m_name)
477 method(self, **met_arg)
478 break
479
480 if self.dataOut.flagNoData:
481 continue
482
483 keyList = list(self.operations2RunDict.keys())
484 keyList.sort()
485
486 for key in keyList:
487
488 self.socketOP = self.sockOp()
489 self.dataOut = self.execOp(self.socketOP, key, self.dataOut)
490
491
492 self.publishProc(self.socket_p, self.dataOut)
493
494
495 print("%s done" %BaseClass.__name__)
496
497 return 0
498
499 def runOp(self):
500
501 while True:
502
503 [self.dest ,self.buffer] = self.funIOrec(self.socket_router)
504
505 self.buffer = BaseClass.run(self, self.buffer, **self.kwargs)
506
507 self.funIOsen(self.socket_router, self.buffer, self.dest)
508
509 print("%s done" %BaseClass.__name__)
510 return 0
511
512
513 def run(self):
514
515 if self.typeProc is "ProcUnit":
516
517 self.socket_p = self.sockPublishing()
518
519 if 'Reader' not in self.dictProcs[self.id].name:
520 self.socket_l = self.sockListening(self.inputId)
521 self.runProc()
522
523 else:
524
525 self.runReader()
526
527 elif self.typeProc is "Operation":
528
529 self.socket_router = self.sockIO()
530
531 self.runOp()
351 532
352 """
353 if not self.isConfig:
354 self.setup(**kwargs)
355
356 raise NotImplementedError
357
358 def close(self):
533 else:
534 raise ValueError("Unknown type")
359 535
360 pass No newline at end of file
536 return 0
537
538 return MPClass No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,953 +1,966
1 1 import itertools
2 2
3 3 import numpy
4 4
5 from .jroproc_base import ProcessingUnit, Operation
5 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
6 6 from schainpy.model.data.jrodata import Spectra
7 7 from schainpy.model.data.jrodata import hildebrand_sekhon
8 from schainpy.utils import log #yong
8 from schainpy.utils import log
9 9
10 @MPDecorator
10 11 class SpectraProc(ProcessingUnit):
11 12
12 def __init__(self, **kwargs):
13 METHODS = {'selectHeights' : ['minHei', 'maxHei'],
14 'selectChannels' : 'channelList',
15 'selectChannelsByIndex': 'channelIndexList',
16 'getBeaconSignal' : ['tauindex', 'channelindex', 'hei_ref'],
17 'selectHeightsByIndex' : ['minIndex', 'maxIndex']
18 }
13 19
14 ProcessingUnit.__init__(self, **kwargs)
20 def __init__(self):#, **kwargs):
21
22 ProcessingUnit.__init__(self)#, **kwargs)
15 23
16 24 self.buffer = None
17 25 self.firstdatatime = None
18 26 self.profIndex = 0
19 27 self.dataOut = Spectra()
20 28 self.id_min = None
21 29 self.id_max = None
30 self.setupReq = False #Agregar a todas las unidades de proc
22 31
23 32 def __updateSpecFromVoltage(self):
24 33
25 34 self.dataOut.timeZone = self.dataIn.timeZone
26 35 self.dataOut.dstFlag = self.dataIn.dstFlag
27 36 self.dataOut.errorCount = self.dataIn.errorCount
28 37 self.dataOut.useLocalTime = self.dataIn.useLocalTime
29 38 try:
30 39 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
31 40 except:
32 41 pass
33 42 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
34 43 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
35 44 self.dataOut.channelList = self.dataIn.channelList
36 45 self.dataOut.heightList = self.dataIn.heightList
37 46 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
38 47
39 48 self.dataOut.nBaud = self.dataIn.nBaud
40 49 self.dataOut.nCode = self.dataIn.nCode
41 50 self.dataOut.code = self.dataIn.code
42 51 self.dataOut.nProfiles = self.dataOut.nFFTPoints
43 52
44 53 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
45 54 self.dataOut.utctime = self.firstdatatime
46 55 # asumo q la data esta decodificada
47 56 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
48 57 # asumo q la data esta sin flip
49 58 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
50 59 self.dataOut.flagShiftFFT = False
51 60
52 61 self.dataOut.nCohInt = self.dataIn.nCohInt
53 62 self.dataOut.nIncohInt = 1
54 63
55 64 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
56 65
57 66 self.dataOut.frequency = self.dataIn.frequency
58 67 self.dataOut.realtime = self.dataIn.realtime
59 68
60 69 self.dataOut.azimuth = self.dataIn.azimuth
61 70 self.dataOut.zenith = self.dataIn.zenith
62 71
63 72 self.dataOut.beam.codeList = self.dataIn.beam.codeList
64 73 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
65 74 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
66 75
67 76 def __getFft(self):
68 77 """
69 78 Convierte valores de Voltaje a Spectra
70 79
71 80 Affected:
72 81 self.dataOut.data_spc
73 82 self.dataOut.data_cspc
74 83 self.dataOut.data_dc
75 84 self.dataOut.heightList
76 85 self.profIndex
77 86 self.buffer
78 87 self.dataOut.flagNoData
79 88 """
80 89 fft_volt = numpy.fft.fft(
81 90 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
82 91 fft_volt = fft_volt.astype(numpy.dtype('complex'))
83 92 dc = fft_volt[:, 0, :]
84 93
85 94 # calculo de self-spectra
86 95 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
87 96 spc = fft_volt * numpy.conjugate(fft_volt)
88 97 spc = spc.real
89 98
90 99 blocksize = 0
91 100 blocksize += dc.size
92 101 blocksize += spc.size
93 102
94 103 cspc = None
95 104 pairIndex = 0
96 105 if self.dataOut.pairsList != None:
97 106 # calculo de cross-spectra
98 107 cspc = numpy.zeros(
99 108 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
100 109 for pair in self.dataOut.pairsList:
101 110 if pair[0] not in self.dataOut.channelList:
102 111 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
103 112 str(pair), str(self.dataOut.channelList)))
104 113 if pair[1] not in self.dataOut.channelList:
105 114 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
106 115 str(pair), str(self.dataOut.channelList)))
107 116
108 117 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
109 118 numpy.conjugate(fft_volt[pair[1], :, :])
110 119 pairIndex += 1
111 120 blocksize += cspc.size
112 121
113 122 self.dataOut.data_spc = spc
114 123 self.dataOut.data_cspc = cspc
115 124 self.dataOut.data_dc = dc
116 125 self.dataOut.blockSize = blocksize
117 126 self.dataOut.flagShiftFFT = True
118 127
119 128 def run(self, nProfiles=None, nFFTPoints=None, pairsList=[], ippFactor=None, shift_fft=False):
120 129
121 130 self.dataOut.flagNoData = True
122 131
123 132 if self.dataIn.type == "Spectra":
124 133 self.dataOut.copy(self.dataIn)
125 134 # if not pairsList:
126 135 # pairsList = itertools.combinations(self.dataOut.channelList, 2)
127 136 # if self.dataOut.data_cspc is not None:
128 137 # self.__selectPairs(pairsList)
129 138 if shift_fft:
130 139 #desplaza a la derecha en el eje 2 determinadas posiciones
131 140 shift = int(self.dataOut.nFFTPoints/2)
132 141 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
133 142
134 143 if self.dataOut.data_cspc is not None:
135 144 #desplaza a la derecha en el eje 2 determinadas posiciones
136 145 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
137
146
138 147 return True
139 148
140 149 if self.dataIn.type == "Voltage":
141 150
142 151 if nFFTPoints == None:
143 152 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
144 153
145 154 if nProfiles == None:
146 155 nProfiles = nFFTPoints
147 156
148 157 if ippFactor == None:
149 158 ippFactor = 1
150 159
151 160 self.dataOut.ippFactor = ippFactor
152 161
153 162 self.dataOut.nFFTPoints = nFFTPoints
154 163 self.dataOut.pairsList = pairsList
155 164
156 165 if self.buffer is None:
157 166 self.buffer = numpy.zeros((self.dataIn.nChannels,
158 167 nProfiles,
159 168 self.dataIn.nHeights),
160 169 dtype='complex')
161 170
162 171 if self.dataIn.flagDataAsBlock:
163 172 # data dimension: [nChannels, nProfiles, nSamples]
164 173 nVoltProfiles = self.dataIn.data.shape[1]
165 174 # nVoltProfiles = self.dataIn.nProfiles
166 175
167 176 if nVoltProfiles == nProfiles:
168 177 self.buffer = self.dataIn.data.copy()
169 178 self.profIndex = nVoltProfiles
170 179
171 180 elif nVoltProfiles < nProfiles:
172 181
173 182 if self.profIndex == 0:
174 183 self.id_min = 0
175 184 self.id_max = nVoltProfiles
176 185
177 186 self.buffer[:, self.id_min:self.id_max,
178 187 :] = self.dataIn.data
179 188 self.profIndex += nVoltProfiles
180 189 self.id_min += nVoltProfiles
181 190 self.id_max += nVoltProfiles
182 191 else:
183 192 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
184 193 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
185 194 self.dataOut.flagNoData = True
186 195 return 0
187 196 else:
188 197 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
189 198 self.profIndex += 1
190 199
191 200 if self.firstdatatime == None:
192 201 self.firstdatatime = self.dataIn.utctime
193 202
194 203 if self.profIndex == nProfiles:
195 204 self.__updateSpecFromVoltage()
196 205 self.__getFft()
197 206
198 207 self.dataOut.flagNoData = False
199 208 self.firstdatatime = None
200 209 self.profIndex = 0
201 210
202 211 return True
203 212
204 213 raise ValueError("The type of input object '%s' is not valid" % (
205 214 self.dataIn.type))
206 215
207 216 def __selectPairs(self, pairsList):
208 217
209 218 if not pairsList:
210 219 return
211 220
212 221 pairs = []
213 222 pairsIndex = []
214 223
215 224 for pair in pairsList:
216 225 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
217 226 continue
218 227 pairs.append(pair)
219 228 pairsIndex.append(pairs.index(pair))
220 229
221 230 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
222 231 self.dataOut.pairsList = pairs
223 232
224 233 return
225 234
226 235 def __selectPairsByChannel(self, channelList=None):
227 236
228 237 if channelList == None:
229 238 return
230 239
231 240 pairsIndexListSelected = []
232 241 for pairIndex in self.dataOut.pairsIndexList:
233 242 # First pair
234 243 if self.dataOut.pairsList[pairIndex][0] not in channelList:
235 244 continue
236 245 # Second pair
237 246 if self.dataOut.pairsList[pairIndex][1] not in channelList:
238 247 continue
239 248
240 249 pairsIndexListSelected.append(pairIndex)
241 250
242 251 if not pairsIndexListSelected:
243 252 self.dataOut.data_cspc = None
244 253 self.dataOut.pairsList = []
245 254 return
246 255
247 256 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
248 257 self.dataOut.pairsList = [self.dataOut.pairsList[i]
249 258 for i in pairsIndexListSelected]
250 259
251 260 return
252 261
253 262 def selectChannels(self, channelList):
254 263
255 264 channelIndexList = []
256 265
257 266 for channel in channelList:
258 267 if channel not in self.dataOut.channelList:
259 268 raise ValueError("Error selecting channels, Channel %d is not valid.\nAvailable channels = %s" % (
260 269 channel, str(self.dataOut.channelList)))
261 270
262 271 index = self.dataOut.channelList.index(channel)
263 272 channelIndexList.append(index)
264 273
265 274 self.selectChannelsByIndex(channelIndexList)
266 275
267 276 def selectChannelsByIndex(self, channelIndexList):
268 277 """
269 278 Selecciona un bloque de datos en base a canales segun el channelIndexList
270 279
271 280 Input:
272 281 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
273 282
274 283 Affected:
275 284 self.dataOut.data_spc
276 285 self.dataOut.channelIndexList
277 286 self.dataOut.nChannels
278 287
279 288 Return:
280 289 None
281 290 """
282 291
283 292 for channelIndex in channelIndexList:
284 293 if channelIndex not in self.dataOut.channelIndexList:
285 294 raise ValueError("Error selecting channels: The value %d in channelIndexList is not valid.\nAvailable channel indexes = " % (
286 295 channelIndex, self.dataOut.channelIndexList))
287 296
288 297 # nChannels = len(channelIndexList)
289 298
290 299 data_spc = self.dataOut.data_spc[channelIndexList, :]
291 300 data_dc = self.dataOut.data_dc[channelIndexList, :]
292 301
293 302 self.dataOut.data_spc = data_spc
294 303 self.dataOut.data_dc = data_dc
295 304
296 305 self.dataOut.channelList = [
297 306 self.dataOut.channelList[i] for i in channelIndexList]
298 307 # self.dataOut.nChannels = nChannels
299 308
300 309 self.__selectPairsByChannel(self.dataOut.channelList)
301 310
302 311 return 1
303 312
304 313 def selectHeights(self, minHei, maxHei):
305 314 """
306 315 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
307 316 minHei <= height <= maxHei
308 317
309 318 Input:
310 319 minHei : valor minimo de altura a considerar
311 320 maxHei : valor maximo de altura a considerar
312 321
313 322 Affected:
314 323 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
315 324
316 325 Return:
317 326 1 si el metodo se ejecuto con exito caso contrario devuelve 0
318 327 """
319 328
320 329 if (minHei > maxHei):
321 330 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (
322 331 minHei, maxHei))
323 332
324 333 if (minHei < self.dataOut.heightList[0]):
325 334 minHei = self.dataOut.heightList[0]
326 335
327 336 if (maxHei > self.dataOut.heightList[-1]):
328 337 maxHei = self.dataOut.heightList[-1]
329 338
330 339 minIndex = 0
331 340 maxIndex = 0
332 341 heights = self.dataOut.heightList
333 342
334 343 inda = numpy.where(heights >= minHei)
335 344 indb = numpy.where(heights <= maxHei)
336 345
337 346 try:
338 347 minIndex = inda[0][0]
339 348 except:
340 349 minIndex = 0
341 350
342 351 try:
343 352 maxIndex = indb[0][-1]
344 353 except:
345 354 maxIndex = len(heights)
346 355
347 356 self.selectHeightsByIndex(minIndex, maxIndex)
348 357
349 358 return 1
350 359
351 360 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
352 361 newheis = numpy.where(
353 362 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
354 363
355 364 if hei_ref != None:
356 365 newheis = numpy.where(self.dataOut.heightList > hei_ref)
357 366
358 367 minIndex = min(newheis[0])
359 368 maxIndex = max(newheis[0])
360 369 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
361 370 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
362 371
363 372 # determina indices
364 373 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
365 374 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
366 375 avg_dB = 10 * \
367 376 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
368 377 beacon_dB = numpy.sort(avg_dB)[-nheis:]
369 378 beacon_heiIndexList = []
370 379 for val in avg_dB.tolist():
371 380 if val >= beacon_dB[0]:
372 381 beacon_heiIndexList.append(avg_dB.tolist().index(val))
373 382
374 383 #data_spc = data_spc[:,:,beacon_heiIndexList]
375 384 data_cspc = None
376 385 if self.dataOut.data_cspc is not None:
377 386 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
378 387 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
379 388
380 389 data_dc = None
381 390 if self.dataOut.data_dc is not None:
382 391 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
383 392 #data_dc = data_dc[:,beacon_heiIndexList]
384 393
385 394 self.dataOut.data_spc = data_spc
386 395 self.dataOut.data_cspc = data_cspc
387 396 self.dataOut.data_dc = data_dc
388 397 self.dataOut.heightList = heightList
389 398 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
390 399
391 400 return 1
392 401
393 402 def selectHeightsByIndex(self, minIndex, maxIndex):
394 403 """
395 404 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
396 405 minIndex <= index <= maxIndex
397 406
398 407 Input:
399 408 minIndex : valor de indice minimo de altura a considerar
400 409 maxIndex : valor de indice maximo de altura a considerar
401 410
402 411 Affected:
403 412 self.dataOut.data_spc
404 413 self.dataOut.data_cspc
405 414 self.dataOut.data_dc
406 415 self.dataOut.heightList
407 416
408 417 Return:
409 418 1 si el metodo se ejecuto con exito caso contrario devuelve 0
410 419 """
411 420
412 421 if (minIndex < 0) or (minIndex > maxIndex):
413 422 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
414 423 minIndex, maxIndex))
415 424
416 425 if (maxIndex >= self.dataOut.nHeights):
417 426 maxIndex = self.dataOut.nHeights - 1
418 427
419 428 # Spectra
420 429 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
421 430
422 431 data_cspc = None
423 432 if self.dataOut.data_cspc is not None:
424 433 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
425 434
426 435 data_dc = None
427 436 if self.dataOut.data_dc is not None:
428 437 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
429 438
430 439 self.dataOut.data_spc = data_spc
431 440 self.dataOut.data_cspc = data_cspc
432 441 self.dataOut.data_dc = data_dc
433 442
434 443 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
435 444
436 445 return 1
437 446
438 447 def removeDC(self, mode=2):
439 448 jspectra = self.dataOut.data_spc
440 449 jcspectra = self.dataOut.data_cspc
441 450
442 451 num_chan = jspectra.shape[0]
443 452 num_hei = jspectra.shape[2]
444 453
445 454 if jcspectra is not None:
446 455 jcspectraExist = True
447 456 num_pairs = jcspectra.shape[0]
448 457 else:
449 458 jcspectraExist = False
450 459
451 460 freq_dc = int(jspectra.shape[1] / 2)
452 461 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
453 462 ind_vel = ind_vel.astype(int)
454 463
455 464 if ind_vel[0] < 0:
456 465 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
457 466
458 467 if mode == 1:
459 468 jspectra[:, freq_dc, :] = (
460 469 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
461 470
462 471 if jcspectraExist:
463 472 jcspectra[:, freq_dc, :] = (
464 473 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
465 474
466 475 if mode == 2:
467 476
468 477 vel = numpy.array([-2, -1, 1, 2])
469 478 xx = numpy.zeros([4, 4])
470 479
471 480 for fil in range(4):
472 481 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
473 482
474 483 xx_inv = numpy.linalg.inv(xx)
475 484 xx_aux = xx_inv[0, :]
476 485
477 486 for ich in range(num_chan):
478 487 yy = jspectra[ich, ind_vel, :]
479 488 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
480 489
481 490 junkid = jspectra[ich, freq_dc, :] <= 0
482 491 cjunkid = sum(junkid)
483 492
484 493 if cjunkid.any():
485 494 jspectra[ich, freq_dc, junkid.nonzero()] = (
486 495 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
487 496
488 497 if jcspectraExist:
489 498 for ip in range(num_pairs):
490 499 yy = jcspectra[ip, ind_vel, :]
491 500 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
492 501
493 502 self.dataOut.data_spc = jspectra
494 503 self.dataOut.data_cspc = jcspectra
495 504
496 505 return 1
497 506
498 507 def removeInterference(self, interf=2, hei_interf=None, nhei_interf=None, offhei_interf=None):
499 508
500 509 jspectra = self.dataOut.data_spc
501 510 jcspectra = self.dataOut.data_cspc
502 511 jnoise = self.dataOut.getNoise()
503 512 num_incoh = self.dataOut.nIncohInt
504 513
505 514 num_channel = jspectra.shape[0]
506 515 num_prof = jspectra.shape[1]
507 516 num_hei = jspectra.shape[2]
508 517
509 518 # hei_interf
510 519 if hei_interf is None:
511 520 count_hei = num_hei / 2 # Como es entero no importa
512 521 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
513 522 hei_interf = numpy.asarray(hei_interf)[0]
514 523 # nhei_interf
515 524 if (nhei_interf == None):
516 525 nhei_interf = 5
517 526 if (nhei_interf < 1):
518 527 nhei_interf = 1
519 528 if (nhei_interf > count_hei):
520 529 nhei_interf = count_hei
521 530 if (offhei_interf == None):
522 531 offhei_interf = 0
523 532
524 533 ind_hei = list(range(num_hei))
525 534 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
526 535 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
527 536 mask_prof = numpy.asarray(list(range(num_prof)))
528 537 num_mask_prof = mask_prof.size
529 538 comp_mask_prof = [0, num_prof / 2]
530 539
531 540 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
532 541 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
533 542 jnoise = numpy.nan
534 543 noise_exist = jnoise[0] < numpy.Inf
535 544
536 545 # Subrutina de Remocion de la Interferencia
537 546 for ich in range(num_channel):
538 547 # Se ordena los espectros segun su potencia (menor a mayor)
539 548 power = jspectra[ich, mask_prof, :]
540 549 power = power[:, hei_interf]
541 550 power = power.sum(axis=0)
542 551 psort = power.ravel().argsort()
543 552
544 553 # Se estima la interferencia promedio en los Espectros de Potencia empleando
545 554 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
546 555 offhei_interf, nhei_interf + offhei_interf))]]]
547 556
548 557 if noise_exist:
549 558 # tmp_noise = jnoise[ich] / num_prof
550 559 tmp_noise = jnoise[ich]
551 560 junkspc_interf = junkspc_interf - tmp_noise
552 561 #junkspc_interf[:,comp_mask_prof] = 0
553 562
554 563 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
555 564 jspc_interf = jspc_interf.transpose()
556 565 # Calculando el espectro de interferencia promedio
557 566 noiseid = numpy.where(
558 567 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
559 568 noiseid = noiseid[0]
560 569 cnoiseid = noiseid.size
561 570 interfid = numpy.where(
562 571 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
563 572 interfid = interfid[0]
564 573 cinterfid = interfid.size
565 574
566 575 if (cnoiseid > 0):
567 576 jspc_interf[noiseid] = 0
568 577
569 578 # Expandiendo los perfiles a limpiar
570 579 if (cinterfid > 0):
571 580 new_interfid = (
572 581 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
573 582 new_interfid = numpy.asarray(new_interfid)
574 583 new_interfid = {x for x in new_interfid}
575 584 new_interfid = numpy.array(list(new_interfid))
576 585 new_cinterfid = new_interfid.size
577 586 else:
578 587 new_cinterfid = 0
579 588
580 589 for ip in range(new_cinterfid):
581 590 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
582 591 jspc_interf[new_interfid[ip]
583 592 ] = junkspc_interf[ind[nhei_interf / 2], new_interfid[ip]]
584 593
585 594 jspectra[ich, :, ind_hei] = jspectra[ich, :,
586 595 ind_hei] - jspc_interf # Corregir indices
587 596
588 597 # Removiendo la interferencia del punto de mayor interferencia
589 598 ListAux = jspc_interf[mask_prof].tolist()
590 599 maxid = ListAux.index(max(ListAux))
591 600
592 601 if cinterfid > 0:
593 602 for ip in range(cinterfid * (interf == 2) - 1):
594 603 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
595 604 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
596 605 cind = len(ind)
597 606
598 607 if (cind > 0):
599 608 jspectra[ich, interfid[ip], ind] = tmp_noise * \
600 609 (1 + (numpy.random.uniform(cind) - 0.5) /
601 610 numpy.sqrt(num_incoh))
602 611
603 612 ind = numpy.array([-2, -1, 1, 2])
604 613 xx = numpy.zeros([4, 4])
605 614
606 615 for id1 in range(4):
607 616 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
608 617
609 618 xx_inv = numpy.linalg.inv(xx)
610 619 xx = xx_inv[:, 0]
611 620 ind = (ind + maxid + num_mask_prof) % num_mask_prof
612 621 yy = jspectra[ich, mask_prof[ind], :]
613 622 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
614 623 yy.transpose(), xx)
615 624
616 625 indAux = (jspectra[ich, :, :] < tmp_noise *
617 626 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
618 627 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
619 628 (1 - 1 / numpy.sqrt(num_incoh))
620 629
621 630 # Remocion de Interferencia en el Cross Spectra
622 631 if jcspectra is None:
623 632 return jspectra, jcspectra
624 633 num_pairs = jcspectra.size / (num_prof * num_hei)
625 634 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
626 635
627 636 for ip in range(num_pairs):
628 637
629 638 #-------------------------------------------
630 639
631 640 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
632 641 cspower = cspower[:, hei_interf]
633 642 cspower = cspower.sum(axis=0)
634 643
635 644 cspsort = cspower.ravel().argsort()
636 645 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
637 646 offhei_interf, nhei_interf + offhei_interf))]]]
638 647 junkcspc_interf = junkcspc_interf.transpose()
639 648 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
640 649
641 650 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
642 651
643 652 median_real = numpy.median(numpy.real(
644 653 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof / 4))]], :]))
645 654 median_imag = numpy.median(numpy.imag(
646 655 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof / 4))]], :]))
647 656 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
648 657 median_real, median_imag)
649 658
650 659 for iprof in range(num_prof):
651 660 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
652 661 jcspc_interf[iprof] = junkcspc_interf[iprof,
653 662 ind[nhei_interf / 2]]
654 663
655 664 # Removiendo la Interferencia
656 665 jcspectra[ip, :, ind_hei] = jcspectra[ip,
657 666 :, ind_hei] - jcspc_interf
658 667
659 668 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
660 669 maxid = ListAux.index(max(ListAux))
661 670
662 671 ind = numpy.array([-2, -1, 1, 2])
663 672 xx = numpy.zeros([4, 4])
664 673
665 674 for id1 in range(4):
666 675 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
667 676
668 677 xx_inv = numpy.linalg.inv(xx)
669 678 xx = xx_inv[:, 0]
670 679
671 680 ind = (ind + maxid + num_mask_prof) % num_mask_prof
672 681 yy = jcspectra[ip, mask_prof[ind], :]
673 682 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
674 683
675 684 # Guardar Resultados
676 685 self.dataOut.data_spc = jspectra
677 686 self.dataOut.data_cspc = jcspectra
678 687
679 688 return 1
680 689
681 690 def setRadarFrequency(self, frequency=None):
682 691
683 692 if frequency != None:
684 693 self.dataOut.frequency = frequency
685 694
686 695 return 1
687 696
688 697 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
689 698 # validacion de rango
690 699 if minHei == None:
691 700 minHei = self.dataOut.heightList[0]
692 701
693 702 if maxHei == None:
694 703 maxHei = self.dataOut.heightList[-1]
695 704
696 705 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
697 706 print('minHei: %.2f is out of the heights range' % (minHei))
698 707 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
699 708 minHei = self.dataOut.heightList[0]
700 709
701 710 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
702 711 print('maxHei: %.2f is out of the heights range' % (maxHei))
703 712 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
704 713 maxHei = self.dataOut.heightList[-1]
705 714
706 715 # validacion de velocidades
707 716 velrange = self.dataOut.getVelRange(1)
708 717
709 718 if minVel == None:
710 719 minVel = velrange[0]
711 720
712 721 if maxVel == None:
713 722 maxVel = velrange[-1]
714 723
715 724 if (minVel < velrange[0]) or (minVel > maxVel):
716 725 print('minVel: %.2f is out of the velocity range' % (minVel))
717 726 print('minVel is setting to %.2f' % (velrange[0]))
718 727 minVel = velrange[0]
719 728
720 729 if (maxVel > velrange[-1]) or (maxVel < minVel):
721 730 print('maxVel: %.2f is out of the velocity range' % (maxVel))
722 731 print('maxVel is setting to %.2f' % (velrange[-1]))
723 732 maxVel = velrange[-1]
724 733
725 734 # seleccion de indices para rango
726 735 minIndex = 0
727 736 maxIndex = 0
728 737 heights = self.dataOut.heightList
729 738
730 739 inda = numpy.where(heights >= minHei)
731 740 indb = numpy.where(heights <= maxHei)
732 741
733 742 try:
734 743 minIndex = inda[0][0]
735 744 except:
736 745 minIndex = 0
737 746
738 747 try:
739 748 maxIndex = indb[0][-1]
740 749 except:
741 750 maxIndex = len(heights)
742 751
743 752 if (minIndex < 0) or (minIndex > maxIndex):
744 753 raise ValueError("some value in (%d,%d) is not valid" % (
745 754 minIndex, maxIndex))
746 755
747 756 if (maxIndex >= self.dataOut.nHeights):
748 757 maxIndex = self.dataOut.nHeights - 1
749 758
750 759 # seleccion de indices para velocidades
751 760 indminvel = numpy.where(velrange >= minVel)
752 761 indmaxvel = numpy.where(velrange <= maxVel)
753 762 try:
754 763 minIndexVel = indminvel[0][0]
755 764 except:
756 765 minIndexVel = 0
757 766
758 767 try:
759 768 maxIndexVel = indmaxvel[0][-1]
760 769 except:
761 770 maxIndexVel = len(velrange)
762 771
763 772 # seleccion del espectro
764 773 data_spc = self.dataOut.data_spc[:,
765 774 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
766 775 # estimacion de ruido
767 776 noise = numpy.zeros(self.dataOut.nChannels)
768 777
769 778 for channel in range(self.dataOut.nChannels):
770 779 daux = data_spc[channel, :, :]
771 780 noise[channel] = hildebrand_sekhon(daux, self.dataOut.nIncohInt)
772 781
773 782 self.dataOut.noise_estimation = noise.copy()
774 783
775 784 return 1
776 785
777
786 @MPDecorator
778 787 class IncohInt(Operation):
779 788
780 789 __profIndex = 0
781 790 __withOverapping = False
782 791
783 792 __byTime = False
784 793 __initime = None
785 794 __lastdatatime = None
786 795 __integrationtime = None
787 796
788 797 __buffer_spc = None
789 798 __buffer_cspc = None
790 799 __buffer_dc = None
791 800
792 801 __dataReady = False
793 802
794 803 __timeInterval = None
795 804
796 805 n = None
797 806
798 def __init__(self, **kwargs):
807 def __init__(self):#, **kwargs):
808
809 Operation.__init__(self)#, **kwargs)
810
799 811
800 Operation.__init__(self, **kwargs)
801 812 # self.isConfig = False
802 813
803 814 def setup(self, n=None, timeInterval=None, overlapping=False):
804 815 """
805 816 Set the parameters of the integration class.
806 817
807 818 Inputs:
808 819
809 820 n : Number of coherent integrations
810 821 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
811 822 overlapping :
812 823
813 824 """
814 825
815 826 self.__initime = None
816 827 self.__lastdatatime = 0
817 828
818 829 self.__buffer_spc = 0
819 830 self.__buffer_cspc = 0
820 831 self.__buffer_dc = 0
821 832
822 833 self.__profIndex = 0
823 834 self.__dataReady = False
824 835 self.__byTime = False
825 836
826 837 if n is None and timeInterval is None:
827 838 raise ValueError("n or timeInterval should be specified ...")
828 839
829 840 if n is not None:
830 841 self.n = int(n)
831 842 else:
832 843 # if (type(timeInterval)!=integer) -> change this line
833 844 self.__integrationtime = int(timeInterval)
834 845 self.n = None
835 846 self.__byTime = True
836 847
837 848 def putData(self, data_spc, data_cspc, data_dc):
838 849 """
839 850 Add a profile to the __buffer_spc and increase in one the __profileIndex
840 851
841 852 """
842 853
843 854 self.__buffer_spc += data_spc
844 855
845 856 if data_cspc is None:
846 857 self.__buffer_cspc = None
847 858 else:
848 859 self.__buffer_cspc += data_cspc
849 860
850 861 if data_dc is None:
851 862 self.__buffer_dc = None
852 863 else:
853 864 self.__buffer_dc += data_dc
854 865
855 866 self.__profIndex += 1
856 867
857 868 return
858 869
859 870 def pushData(self):
860 871 """
861 872 Return the sum of the last profiles and the profiles used in the sum.
862 873
863 874 Affected:
864 875
865 876 self.__profileIndex
866 877
867 878 """
868 879
869 880 data_spc = self.__buffer_spc
870 881 data_cspc = self.__buffer_cspc
871 882 data_dc = self.__buffer_dc
872 883 n = self.__profIndex
873 884
874 885 self.__buffer_spc = 0
875 886 self.__buffer_cspc = 0
876 887 self.__buffer_dc = 0
877 888 self.__profIndex = 0
878 889
879 890 return data_spc, data_cspc, data_dc, n
880 891
881 892 def byProfiles(self, *args):
882 893
883 894 self.__dataReady = False
884 895 avgdata_spc = None
885 896 avgdata_cspc = None
886 897 avgdata_dc = None
887 898
888 899 self.putData(*args)
889 900
890 901 if self.__profIndex == self.n:
891 902
892 903 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
893 904 self.n = n
894 905 self.__dataReady = True
895 906
896 907 return avgdata_spc, avgdata_cspc, avgdata_dc
897 908
898 909 def byTime(self, datatime, *args):
899 910
900 911 self.__dataReady = False
901 912 avgdata_spc = None
902 913 avgdata_cspc = None
903 914 avgdata_dc = None
904 915
905 916 self.putData(*args)
906 917
907 918 if (datatime - self.__initime) >= self.__integrationtime:
908 919 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
909 920 self.n = n
910 921 self.__dataReady = True
911 922
912 923 return avgdata_spc, avgdata_cspc, avgdata_dc
913 924
914 925 def integrate(self, datatime, *args):
915 926
916 927 if self.__profIndex == 0:
917 928 self.__initime = datatime
918 929
919 930 if self.__byTime:
920 931 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
921 932 datatime, *args)
922 933 else:
923 934 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
924 935
925 936 if not self.__dataReady:
926 937 return None, None, None, None
927 938
928 939 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
929 940
930 941 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
931 942 if n == 1:
932 943 return
933
944
934 945 dataOut.flagNoData = True
935 946
936 947 if not self.isConfig:
937 948 self.setup(n, timeInterval, overlapping)
938 949 self.isConfig = True
939 950
940 951 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
941 952 dataOut.data_spc,
942 953 dataOut.data_cspc,
943 954 dataOut.data_dc)
944 955
945 956 if self.__dataReady:
946 957
947 958 dataOut.data_spc = avgdata_spc
948 959 dataOut.data_cspc = avgdata_cspc
949 960 dataOut.data_dc = avgdata_dc
950 961
951 962 dataOut.nIncohInt *= self.n
952 963 dataOut.utctime = avgdatatime
953 dataOut.flagNoData = False No newline at end of file
964 dataOut.flagNoData = False
965
966 return dataOut No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now