##// END OF EJS Templates
schain-jc
RSS
Merge branch 'v3.0-devel' of http://jro-dev.igp.gob.pe/rhodecode/schain into v3.0-devel
Juan C. Espinoza -
r1175:7e36d2e90f1d merge
parent child
Show More
Side by Side Unified
Context file:
r1175:7e36d2e90f1d Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1342 +1,1333
1 1 '''
2 Updated on January , 2018, for multiprocessing purposes
3 Author: Sergio Cortez
2 4 Created on September , 2012
3 @author:
4 5 '''
5
6 from platform import python_version
6 7 import sys
7 8 import ast
8 9 import datetime
9 10 import traceback
10 11 import math
11 12 import time
13 import zmq
12 14 from multiprocessing import Process, cpu_count
13 15
14 16 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 17 from xml.dom import minidom
16 18
17 import schainpy
19
18 20 from schainpy.admin import Alarm, SchainWarning
19 from schainpy.model import *
21
22 ### Temporary imports!!!
23 # from schainpy.model import *
24 from schainpy.model.io import *
25 from schainpy.model.graphics import *
26 from schainpy.model.proc.jroproc_base import *
27 from schainpy.model.proc.bltrproc_parameters import *
28 from schainpy.model.proc.jroproc_spectra import *
29 from schainpy.model.proc.jroproc_voltage import *
30 from schainpy.model.proc.jroproc_parameters import *
31 from schainpy.model.utils.jroutils_publish import *
20 32 from schainpy.utils import log
33 ###
21 34
22 35 DTYPES = {
23 36 'Voltage': '.r',
24 37 'Spectra': '.pdata'
25 38 }
26 39
27 40
28 41 def MPProject(project, n=cpu_count()):
29 42 '''
30 43 Project wrapper to run schain in n processes
31 44 '''
32 45
33 46 rconf = project.getReadUnitObj()
34 47 op = rconf.getOperationObj('run')
35 48 dt1 = op.getParameterValue('startDate')
36 49 dt2 = op.getParameterValue('endDate')
37 50 tm1 = op.getParameterValue('startTime')
38 51 tm2 = op.getParameterValue('endTime')
39 52 days = (dt2 - dt1).days
40 53
41 54 for day in range(days + 1):
42 55 skip = 0
43 56 cursor = 0
44 57 processes = []
45 58 dt = dt1 + datetime.timedelta(day)
46 59 dt_str = dt.strftime('%Y/%m/%d')
47 60 reader = JRODataReader()
48 61 paths, files = reader.searchFilesOffLine(path=rconf.path,
49 62 startDate=dt,
50 63 endDate=dt,
51 64 startTime=tm1,
52 65 endTime=tm2,
53 66 ext=DTYPES[rconf.datatype])
54 67 nFiles = len(files)
55 68 if nFiles == 0:
56 69 continue
57 70 skip = int(math.ceil(nFiles / n))
58 71 while nFiles > cursor * skip:
59 72 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
60 73 skip=skip)
61 74 p = project.clone()
62 75 p.start()
63 76 processes.append(p)
64 77 cursor += 1
65 78
66 79 def beforeExit(exctype, value, trace):
67 80 for process in processes:
68 81 process.terminate()
69 82 process.join()
70 83 print(traceback.print_tb(trace))
71 84
72 85 sys.excepthook = beforeExit
73 86
74 87 for process in processes:
75 88 process.join()
76 89 process.terminate()
77 90
78 91 time.sleep(3)
79 92
80
81 93 class ParameterConf():
82 94
83 95 id = None
84 96 name = None
85 97 value = None
86 98 format = None
87 99
88 100 __formated_value = None
89 101
90 102 ELEMENTNAME = 'Parameter'
91 103
92 104 def __init__(self):
93 105
94 106 self.format = 'str'
95 107
96 108 def getElementName(self):
97 109
98 110 return self.ELEMENTNAME
99 111
100 112 def getValue(self):
101 113
102 114 value = self.value
103 115 format = self.format
104 116
105 117 if self.__formated_value != None:
106 118
107 119 return self.__formated_value
108 120
109 121 if format == 'obj':
110 122 return value
111 123
112 124 if format == 'str':
113 125 self.__formated_value = str(value)
114 126 return self.__formated_value
115 127
116 128 if value == '':
117 129 raise ValueError('%s: This parameter value is empty' % self.name)
118 130
119 131 if format == 'list':
120 132 strList = value.split(',')
121 133
122 134 self.__formated_value = strList
123 135
124 136 return self.__formated_value
125 137
126 138 if format == 'intlist':
127 139 '''
128 140 Example:
129 141 value = (0,1,2)
130 142 '''
131 143
132 144 new_value = ast.literal_eval(value)
133 145
134 146 if type(new_value) not in (tuple, list):
135 147 new_value = [int(new_value)]
136 148
137 149 self.__formated_value = new_value
138 150
139 151 return self.__formated_value
140 152
141 153 if format == 'floatlist':
142 154 '''
143 155 Example:
144 156 value = (0.5, 1.4, 2.7)
145 157 '''
146 158
147 159 new_value = ast.literal_eval(value)
148 160
149 161 if type(new_value) not in (tuple, list):
150 162 new_value = [float(new_value)]
151 163
152 164 self.__formated_value = new_value
153 165
154 166 return self.__formated_value
155 167
156 168 if format == 'date':
157 169 strList = value.split('/')
158 170 intList = [int(x) for x in strList]
159 171 date = datetime.date(intList[0], intList[1], intList[2])
160 172
161 173 self.__formated_value = date
162 174
163 175 return self.__formated_value
164 176
165 177 if format == 'time':
166 178 strList = value.split(':')
167 179 intList = [int(x) for x in strList]
168 180 time = datetime.time(intList[0], intList[1], intList[2])
169 181
170 182 self.__formated_value = time
171 183
172 184 return self.__formated_value
173 185
174 186 if format == 'pairslist':
175 187 '''
176 188 Example:
177 189 value = (0,1),(1,2)
178 190 '''
179 191
180 192 new_value = ast.literal_eval(value)
181 193
182 194 if type(new_value) not in (tuple, list):
183 195 raise ValueError('%s has to be a tuple or list of pairs' % value)
184 196
185 197 if type(new_value[0]) not in (tuple, list):
186 198 if len(new_value) != 2:
187 199 raise ValueError('%s has to be a tuple or list of pairs' % value)
188 200 new_value = [new_value]
189 201
190 202 for thisPair in new_value:
191 203 if len(thisPair) != 2:
192 204 raise ValueError('%s has to be a tuple or list of pairs' % value)
193 205
194 206 self.__formated_value = new_value
195 207
196 208 return self.__formated_value
197 209
198 210 if format == 'multilist':
199 211 '''
200 212 Example:
201 213 value = (0,1,2),(3,4,5)
202 214 '''
203 215 multiList = ast.literal_eval(value)
204 216
205 217 if type(multiList[0]) == int:
206 218 multiList = ast.literal_eval('(' + value + ')')
207 219
208 220 self.__formated_value = multiList
209 221
210 222 return self.__formated_value
211 223
212 224 if format == 'bool':
213 225 value = int(value)
214 226
215 227 if format == 'int':
216 228 value = float(value)
217 229
218 230 format_func = eval(format)
219 231
220 232 self.__formated_value = format_func(value)
221 233
222 234 return self.__formated_value
223 235
224 236 def updateId(self, new_id):
225 237
226 238 self.id = str(new_id)
227 239
228 240 def setup(self, id, name, value, format='str'):
229 241 self.id = str(id)
230 242 self.name = name
231 243 if format == 'obj':
232 244 self.value = value
233 245 else:
234 246 self.value = str(value)
235 247 self.format = str.lower(format)
236 248
237 249 self.getValue()
238 250
239 251 return 1
240 252
241 253 def update(self, name, value, format='str'):
242 254
243 255 self.name = name
244 256 self.value = str(value)
245 257 self.format = format
246 258
247 259 def makeXml(self, opElement):
248 260 if self.name not in ('queue',):
249 261 parmElement = SubElement(opElement, self.ELEMENTNAME)
250 262 parmElement.set('id', str(self.id))
251 263 parmElement.set('name', self.name)
252 264 parmElement.set('value', self.value)
253 265 parmElement.set('format', self.format)
254 266
255 267 def readXml(self, parmElement):
256 268
257 269 self.id = parmElement.get('id')
258 270 self.name = parmElement.get('name')
259 271 self.value = parmElement.get('value')
260 272 self.format = str.lower(parmElement.get('format'))
261 273
262 274 # Compatible with old signal chain version
263 275 if self.format == 'int' and self.name == 'idfigure':
264 276 self.name = 'id'
265 277
266 278 def printattr(self):
267 279
268 280 print('Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format))
269 281
270
271 282 class OperationConf():
272 283
273 284 id = None
274 285 name = None
275 286 priority = None
276 287 type = None
277 288
278 289 parmConfObjList = []
279 290
280 291 ELEMENTNAME = 'Operation'
281 292
282 293 def __init__(self):
283 294
284 295 self.id = '0'
285 296 self.name = None
286 297 self.priority = None
287 self.type = 'self'
298 self.topic = None
288 299
289 300 def __getNewId(self):
290 301
291 302 return int(self.id) * 10 + len(self.parmConfObjList) + 1
292 303
304 def getId(self):
305 return self.id
306
293 307 def updateId(self, new_id):
294 308
295 309 self.id = str(new_id)
296 310
297 311 n = 1
298 312 for parmObj in self.parmConfObjList:
299 313
300 314 idParm = str(int(new_id) * 10 + n)
301 315 parmObj.updateId(idParm)
302 316
303 317 n += 1
304 318
305 319 def getElementName(self):
306 320
307 321 return self.ELEMENTNAME
308 322
309 323 def getParameterObjList(self):
310 324
311 325 return self.parmConfObjList
312 326
313 327 def getParameterObj(self, parameterName):
314 328
315 329 for parmConfObj in self.parmConfObjList:
316 330
317 331 if parmConfObj.name != parameterName:
318 332 continue
319 333
320 334 return parmConfObj
321 335
322 336 return None
323 337
324 338 def getParameterObjfromValue(self, parameterValue):
325 339
326 340 for parmConfObj in self.parmConfObjList:
327 341
328 342 if parmConfObj.getValue() != parameterValue:
329 343 continue
330 344
331 345 return parmConfObj.getValue()
332 346
333 347 return None
334 348
335 349 def getParameterValue(self, parameterName):
336 350
337 351 parameterObj = self.getParameterObj(parameterName)
338 352
339 353 # if not parameterObj:
340 354 # return None
341 355
342 356 value = parameterObj.getValue()
343 357
344 358 return value
345 359
346 360 def getKwargs(self):
347 361
348 362 kwargs = {}
349 363
350 364 for parmConfObj in self.parmConfObjList:
351 365 if self.name == 'run' and parmConfObj.name == 'datatype':
352 366 continue
353 367
354 368 kwargs[parmConfObj.name] = parmConfObj.getValue()
355 369
356 370 return kwargs
357 371
358 372 def setup(self, id, name, priority, type):
359 373
360 374 self.id = str(id)
361 375 self.name = name
362 376 self.type = type
363 377 self.priority = priority
364
365 378 self.parmConfObjList = []
366 379
367 380 def removeParameters(self):
368 381
369 382 for obj in self.parmConfObjList:
370 383 del obj
371 384
372 385 self.parmConfObjList = []
373 386
374 387 def addParameter(self, name, value, format='str'):
375 388
376 389 if value is None:
377 390 return None
378 391 id = self.__getNewId()
379 392
380 393 parmConfObj = ParameterConf()
381 394 if not parmConfObj.setup(id, name, value, format):
382 395 return None
383 396
384 397 self.parmConfObjList.append(parmConfObj)
385 398
386 399 return parmConfObj
387 400
388 401 def changeParameter(self, name, value, format='str'):
389 402
390 403 parmConfObj = self.getParameterObj(name)
391 404 parmConfObj.update(name, value, format)
392 405
393 406 return parmConfObj
394 407
395 408 def makeXml(self, procUnitElement):
396 409
397 410 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
398 411 opElement.set('id', str(self.id))
399 412 opElement.set('name', self.name)
400 413 opElement.set('type', self.type)
401 414 opElement.set('priority', str(self.priority))
402 415
403 416 for parmConfObj in self.parmConfObjList:
404 417 parmConfObj.makeXml(opElement)
405 418
406 419 def readXml(self, opElement):
407 420
408 421 self.id = opElement.get('id')
409 422 self.name = opElement.get('name')
410 423 self.type = opElement.get('type')
411 424 self.priority = opElement.get('priority')
412 425
413 426 # Compatible with old signal chain version
414 427 # Use of 'run' method instead 'init'
415 428 if self.type == 'self' and self.name == 'init':
416 429 self.name = 'run'
417 430
418 431 self.parmConfObjList = []
419 432
420 433 parmElementList = opElement.iter(ParameterConf().getElementName())
421 434
422 435 for parmElement in parmElementList:
423 436 parmConfObj = ParameterConf()
424 437 parmConfObj.readXml(parmElement)
425 438
426 439 # Compatible with old signal chain version
427 440 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
428 441 if self.type != 'self' and self.name == 'Plot':
429 442 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
430 443 self.name = parmConfObj.value
431 444 continue
432 445
433 446 self.parmConfObjList.append(parmConfObj)
434 447
435 448 def printattr(self):
436 449
437 450 print('%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
438 451 self.id,
439 452 self.name,
440 453 self.type,
441 454 self.priority))
442 455
443 456 for parmConfObj in self.parmConfObjList:
444 457 parmConfObj.printattr()
445 458
446 def createObject(self, plotter_queue=None):
447
448 if self.type == 'self':
449 raise ValueError('This operation type cannot be created')
459 def createObject(self):
450 460
451 if self.type == 'plotter':
452 if not plotter_queue:
453 raise ValueError('plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)')
454
455 opObj = Plotter(self.name, plotter_queue)
461 className = eval(self.name)
462 kwargs = self.getKwargs()
456 463
457 if self.type == 'external' or self.type == 'other':
464 opObj = className(self.id, **kwargs)
458 465
459 className = eval(self.name)
460 kwargs = self.getKwargs()
466 opObj.start()
461 467
462 opObj = className(**kwargs)
468 print(' Operation created')
463 469
464 470 return opObj
465 471
466
467 472 class ProcUnitConf():
468 473
469 474 id = None
470 475 name = None
471 476 datatype = None
472 477 inputId = None
473 478 parentId = None
474 479
475 480 opConfObjList = []
476 481
477 482 procUnitObj = None
478 483 opObjList = []
479 484
480 485 ELEMENTNAME = 'ProcUnit'
481 486
482 487 def __init__(self):
483 488
484 489 self.id = None
485 490 self.datatype = None
486 491 self.name = None
487 self.inputId = None
492 self.inputId = None
488 493
489 494 self.opConfObjList = []
490 495
491 496 self.procUnitObj = None
492 497 self.opObjDict = {}
493 498
494 499 def __getPriority(self):
495 500
496 501 return len(self.opConfObjList) + 1
497 502
498 503 def __getNewId(self):
499 504
500 505 return int(self.id) * 10 + len(self.opConfObjList) + 1
501 506
502 507 def getElementName(self):
503 508
504 509 return self.ELEMENTNAME
505 510
506 511 def getId(self):
507 512
508 513 return self.id
509 514
510 def updateId(self, new_id, parentId=parentId):
511
515 def updateId(self, new_id, parentId=parentId):
516 '''
512 517 new_id = int(parentId) * 10 + (int(self.id) % 10)
513 518 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
514 519
515 520 # If this proc unit has not inputs
516 if self.inputId == '0':
517 new_inputId = 0
521 #if self.inputId == '0':
522 #new_inputId = 0
518 523
519 524 n = 1
520 525 for opConfObj in self.opConfObjList:
521 526
522 527 idOp = str(int(new_id) * 10 + n)
523 528 opConfObj.updateId(idOp)
524 529
525 530 n += 1
526 531
527 532 self.parentId = str(parentId)
528 533 self.id = str(new_id)
529 self.inputId = str(new_inputId)
530
534 #self.inputId = str(new_inputId)
535 '''
536 n = 1
531 537 def getInputId(self):
532 538
533 539 return self.inputId
534 540
535 541 def getOperationObjList(self):
536 542
537 543 return self.opConfObjList
538 544
539 545 def getOperationObj(self, name=None):
540 546
541 547 for opConfObj in self.opConfObjList:
542 548
543 549 if opConfObj.name != name:
544 550 continue
545 551
546 552 return opConfObj
547 553
548 554 return None
549 555
550 556 def getOpObjfromParamValue(self, value=None):
551 557
552 558 for opConfObj in self.opConfObjList:
553 559 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
554 560 continue
555 561 return opConfObj
556 562 return None
557 563
558 564 def getProcUnitObj(self):
559 565
560 566 return self.procUnitObj
561 567
562 568 def setup(self, id, name, datatype, inputId, parentId=None):
563
569 '''
570 id sera el topico a publicar
571 inputId sera el topico a subscribirse
572 '''
573
564 574 # Compatible with old signal chain version
565 575 if datatype == None and name == None:
566 576 raise ValueError('datatype or name should be defined')
567 577
578 #Definir una condicion para inputId cuando sea 0
579
568 580 if name == None:
569 581 if 'Proc' in datatype:
570 582 name = datatype
571 583 else:
572 584 name = '%sProc' % (datatype)
573 585
574 586 if datatype == None:
575 587 datatype = name.replace('Proc', '')
576 588
577 589 self.id = str(id)
578 590 self.name = name
579 591 self.datatype = datatype
580 self.inputId = inputId
592 self.inputId = inputId
581 593 self.parentId = parentId
582
583 594 self.opConfObjList = []
584 595
585 self.addOperation(name='run', optype='self')
596 self.addOperation(name='run', optype='self')
586 597
587 598 def removeOperations(self):
588 599
589 600 for obj in self.opConfObjList:
590 601 del obj
591 602
592 603 self.opConfObjList = []
593 604 self.addOperation(name='run')
594 605
595 606 def addParameter(self, **kwargs):
596 607 '''
597 608 Add parameters to 'run' operation
598 609 '''
599 610 opObj = self.opConfObjList[0]
600 611
601 612 opObj.addParameter(**kwargs)
602 613
603 614 return opObj
604 615
605 def addOperation(self, name, optype='self'):
616 def addOperation(self, name, optype = 'self'):
617 '''
618 Actualizacion - > proceso comunicacion
619 En el caso de optype='self', elminar. DEfinir comuncacion IPC -> Topic
620 definir el tipoc de socket o comunicacion ipc++
621
622 '''
606 623
607 624 id = self.__getNewId()
608 priority = self.__getPriority()
625 priority = self.__getPriority() # Sin mucho sentido, pero puede usarse
609 626
610 627 opConfObj = OperationConf()
611 628 opConfObj.setup(id, name=name, priority=priority, type=optype)
612 629
613 630 self.opConfObjList.append(opConfObj)
614 631
615 632 return opConfObj
616 633
617 634 def makeXml(self, projectElement):
618 635
619 636 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
620 637 procUnitElement.set('id', str(self.id))
621 638 procUnitElement.set('name', self.name)
622 639 procUnitElement.set('datatype', self.datatype)
623 640 procUnitElement.set('inputId', str(self.inputId))
624 641
625 642 for opConfObj in self.opConfObjList:
626 643 opConfObj.makeXml(procUnitElement)
627 644
628 645 def readXml(self, upElement):
629 646
630 647 self.id = upElement.get('id')
631 648 self.name = upElement.get('name')
632 649 self.datatype = upElement.get('datatype')
633 650 self.inputId = upElement.get('inputId')
634 651
635 652 if self.ELEMENTNAME == 'ReadUnit':
636 653 self.datatype = self.datatype.replace('Reader', '')
637 654
638 655 if self.ELEMENTNAME == 'ProcUnit':
639 656 self.datatype = self.datatype.replace('Proc', '')
640 657
641 658 if self.inputId == 'None':
642 659 self.inputId = '0'
643 660
644 661 self.opConfObjList = []
645 662
646 663 opElementList = upElement.iter(OperationConf().getElementName())
647 664
648 665 for opElement in opElementList:
649 666 opConfObj = OperationConf()
650 667 opConfObj.readXml(opElement)
651 668 self.opConfObjList.append(opConfObj)
652 669
653 670 def printattr(self):
654 671
655 672 print('%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
656 673 self.id,
657 674 self.name,
658 675 self.datatype,
659 676 self.inputId))
660 677
661 678 for opConfObj in self.opConfObjList:
662 679 opConfObj.printattr()
663 680
664 681 def getKwargs(self):
665 682
666 683 opObj = self.opConfObjList[0]
667 684 kwargs = opObj.getKwargs()
668 685
669 686 return kwargs
670 687
671 def createObjects(self, plotter_queue=None):
688 def createObjects(self, dictUnits):
689 '''
690 Instancia de unidades de procesamiento.
672 691
692 '''
673 693 className = eval(self.name)
674 694 kwargs = self.getKwargs()
675 procUnitObj = className(**kwargs)
695 procUnitObj = className(self.id, self.inputId, dictUnits, **kwargs) # necesitan saber su id y su entrada por fines de ipc
696
676 697
677 698 for opConfObj in self.opConfObjList:
678 699
679 700 if opConfObj.type == 'self' and self.name == 'run':
680 701 continue
681 702 elif opConfObj.type == 'self':
682 703 procUnitObj.addOperationKwargs(
683 704 opConfObj.id, **opConfObj.getKwargs())
684 705 continue
685
686 opObj = opConfObj.createObject(plotter_queue)
687
688 self.opObjDict[opConfObj.id] = opObj
689
690 procUnitObj.addOperation(opObj, opConfObj.id)
706 print("Creating operation process:", opConfObj.name, "for", self.name)
707 opObj = opConfObj.createObject()
708
709
710 #self.opObjDict[opConfObj.id] = opObj.name
711
712 procUnitObj.addOperation(opConfObj.name, opConfObj.id)
713
714 procUnitObj.start()
691 715
692 716 self.procUnitObj = procUnitObj
717
693 718
694 719 return procUnitObj
695 720
696 721 def run(self):
697
698 is_ok = False
699
722
723 is_ok = True
724 """
700 725 for opConfObj in self.opConfObjList:
701 726
702 727 kwargs = {}
703 728 for parmConfObj in opConfObj.getParameterObjList():
704 729 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
705 730 continue
706 731
707 732 kwargs[parmConfObj.name] = parmConfObj.getValue()
708 733
709 734 sts = self.procUnitObj.call(opType=opConfObj.type,
710 735 opName=opConfObj.name,
711 736 opId=opConfObj.id)
712 737
713 738 is_ok = is_ok or sts
714
739
740 """
715 741 return is_ok
716
742
743
717 744 def close(self):
718 745
719 746 for opConfObj in self.opConfObjList:
720 747 if opConfObj.type == 'self':
721 748 continue
722 749
723 750 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
724 751 opObj.close()
725 752
726 753 self.procUnitObj.close()
727 754
728 755 return
729 756
730 757
731 758 class ReadUnitConf(ProcUnitConf):
732 759
733 760 path = None
734 761 startDate = None
735 762 endDate = None
736 763 startTime = None
737 764 endTime = None
738 765
739 766 ELEMENTNAME = 'ReadUnit'
740 767
741 768 def __init__(self):
742 769
743 770 self.id = None
744 771 self.datatype = None
745 772 self.name = None
746 773 self.inputId = None
747 774
748 775 self.parentId = None
749 776
750 777 self.opConfObjList = []
751 778 self.opObjList = []
752 779
753 780 def getElementName(self):
754 781
755 return self.ELEMENTNAME
756
782 return self.ELEMENTNAME
783
757 784 def setup(self, id, name, datatype, path='', startDate='', endDate='',
758 785 startTime='', endTime='', parentId=None, server=None, **kwargs):
759 786
787
788 '''
789 *****el id del proceso sera el Topico
790
791 Adicion de {topic}, si no esta presente -> error
792 kwargs deben ser trasmitidos en la instanciacion
793
794 '''
795
760 796 # Compatible with old signal chain version
761 797 if datatype == None and name == None:
762 798 raise ValueError('datatype or name should be defined')
763 799 if name == None:
764 800 if 'Reader' in datatype:
765 801 name = datatype
766 802 datatype = name.replace('Reader','')
767 803 else:
768 804 name = '{}Reader'.format(datatype)
769 805 if datatype == None:
770 806 if 'Reader' in name:
771 807 datatype = name.replace('Reader','')
772 808 else:
773 809 datatype = name
774 810 name = '{}Reader'.format(name)
775 811
776 812 self.id = id
777 813 self.name = name
778 814 self.datatype = datatype
779 815 if path != '':
780 816 self.path = os.path.abspath(path)
781 817 self.startDate = startDate
782 818 self.endDate = endDate
783 819 self.startTime = startTime
784 820 self.endTime = endTime
785 821 self.inputId = '0'
786 822 self.parentId = parentId
787 823 self.server = server
788 824 self.addRunOperation(**kwargs)
789 825
790 826 def update(self, **kwargs):
791 827
792 828 if 'datatype' in kwargs:
793 829 datatype = kwargs.pop('datatype')
794 830 if 'Reader' in datatype:
795 831 self.name = datatype
796 832 else:
797 833 self.name = '%sReader' % (datatype)
798 834 self.datatype = self.name.replace('Reader', '')
799 835
800 836 attrs = ('path', 'startDate', 'endDate',
801 837 'startTime', 'endTime', 'parentId')
802 838
803 839 for attr in attrs:
804 840 if attr in kwargs:
805 841 setattr(self, attr, kwargs.pop(attr))
806 842
807 843 self.inputId = '0'
808 844 self.updateRunOperation(**kwargs)
809 845
810 846 def removeOperations(self):
811 847
812 848 for obj in self.opConfObjList:
813 849 del obj
814 850
815 851 self.opConfObjList = []
816 852
817 def addRunOperation(self, **kwargs):
853 def addRunOperation(self, **kwargs):
818 854
819 opObj = self.addOperation(name='run', optype='self')
855 opObj = self.addOperation(name='run', optype='self')
820 856
821 857 if self.server is None:
822 858 opObj.addParameter(
823 859 name='datatype', value=self.datatype, format='str')
824 860 opObj.addParameter(name='path', value=self.path, format='str')
825 861 opObj.addParameter(
826 862 name='startDate', value=self.startDate, format='date')
827 863 opObj.addParameter(
828 864 name='endDate', value=self.endDate, format='date')
829 865 opObj.addParameter(
830 866 name='startTime', value=self.startTime, format='time')
831 867 opObj.addParameter(
832 868 name='endTime', value=self.endTime, format='time')
833 869
834 870 for key, value in list(kwargs.items()):
835 871 opObj.addParameter(name=key, value=value,
836 872 format=type(value).__name__)
837 873 else:
838 874 opObj.addParameter(name='server', value=self.server, format='str')
839 875
840 876 return opObj
841 877
842 878 def updateRunOperation(self, **kwargs):
843 879
844 880 opObj = self.getOperationObj(name='run')
845 881 opObj.removeParameters()
846 882
847 883 opObj.addParameter(name='datatype', value=self.datatype, format='str')
848 884 opObj.addParameter(name='path', value=self.path, format='str')
849 885 opObj.addParameter(
850 886 name='startDate', value=self.startDate, format='date')
851 887 opObj.addParameter(name='endDate', value=self.endDate, format='date')
852 888 opObj.addParameter(
853 889 name='startTime', value=self.startTime, format='time')
854 890 opObj.addParameter(name='endTime', value=self.endTime, format='time')
855 891
856 892 for key, value in list(kwargs.items()):
857 893 opObj.addParameter(name=key, value=value,
858 894 format=type(value).__name__)
859 895
860 896 return opObj
861 897
862 898 def readXml(self, upElement):
863 899
864 900 self.id = upElement.get('id')
865 901 self.name = upElement.get('name')
866 902 self.datatype = upElement.get('datatype')
867 903 self.inputId = upElement.get('inputId')
868 904
869 905 if self.ELEMENTNAME == 'ReadUnit':
870 906 self.datatype = self.datatype.replace('Reader', '')
871 907
872 908 if self.inputId == 'None':
873 909 self.inputId = '0'
874 910
875 911 self.opConfObjList = []
876 912
877 913 opElementList = upElement.iter(OperationConf().getElementName())
878 914
879 915 for opElement in opElementList:
880 916 opConfObj = OperationConf()
881 917 opConfObj.readXml(opElement)
882 918 self.opConfObjList.append(opConfObj)
883 919
884 920 if opConfObj.name == 'run':
885 921 self.path = opConfObj.getParameterValue('path')
886 922 self.startDate = opConfObj.getParameterValue('startDate')
887 923 self.endDate = opConfObj.getParameterValue('endDate')
888 924 self.startTime = opConfObj.getParameterValue('startTime')
889 925 self.endTime = opConfObj.getParameterValue('endTime')
890 926
891 927
892 928 class Project(Process):
893 929
894 930 id = None
895 # name = None
896 931 description = None
897 932 filename = None
898 933
899 934 procUnitConfObjDict = None
900 935
901 936 ELEMENTNAME = 'Project'
902 937
903 plotterQueue = None
938
904 939
905 def __init__(self, plotter_queue=None):
940 def __init__(self):
906 941
907 942 Process.__init__(self)
908 self.id = None
943 self.id = None
909 944 self.description = None
910 945 self.email = None
911 946 self.alarm = None
912 self.plotterQueue = plotter_queue
913 947 self.procUnitConfObjDict = {}
914 948
915 949 def __getNewId(self):
916 950
917 951 idList = list(self.procUnitConfObjDict.keys())
918 952
919 953 id = int(self.id) * 10
920 954
921 955 while True:
922 956 id += 1
923 957
924 958 if str(id) in idList:
925 959 continue
926 960
927 961 break
928 962
929 963 return str(id)
930 964
931 965 def getElementName(self):
932 966
933 967 return self.ELEMENTNAME
934 968
935 969 def getId(self):
936 970
937 971 return self.id
938 972
939 973 def updateId(self, new_id):
940 974
941 975 self.id = str(new_id)
942 976
943 977 keyList = list(self.procUnitConfObjDict.keys())
944 978 keyList.sort()
945 979
946 980 n = 1
947 981 newProcUnitConfObjDict = {}
948 982
949 983 for procKey in keyList:
950 984
951 985 procUnitConfObj = self.procUnitConfObjDict[procKey]
952 986 idProcUnit = str(int(self.id) * 10 + n)
953 987 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
954 988 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
955 989 n += 1
956 990
957 991 self.procUnitConfObjDict = newProcUnitConfObjDict
958 992
959 993 def setup(self, id, name='', description='', email=None, alarm=[]):
960 994
961 print()
995 print(' ')
962 996 print('*' * 60)
963 print(' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__)
997 print('* Starting SIGNAL CHAIN PROCESSING (Multiprocessing) v%s *' % schainpy.__version__)
964 998 print('*' * 60)
965 print()
999 print("* Python " + python_version() + " *")
1000 print('*' * 19)
1001 print(' ')
966 1002 self.id = str(id)
967 self.description = description
1003 self.description = description
968 1004 self.email = email
969 1005 self.alarm = alarm
970 1006
971 1007 def update(self, **kwargs):
972 1008
973 1009 for key, value in list(kwargs.items()):
974 1010 setattr(self, key, value)
975 1011
976 1012 def clone(self):
977 1013
978 1014 p = Project()
979 1015 p.procUnitConfObjDict = self.procUnitConfObjDict
980 1016 return p
981 1017
982 1018 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
983 1019
1020 '''
1021 Actualizacion:
1022 Se agrego un nuevo argumento: topic -relativo a la forma de comunicar los procesos simultaneos
1023
1024 * El id del proceso sera el topico al que se deben subscribir los procUnits para recibir la informacion(data)
1025
1026 '''
1027
984 1028 if id is None:
985 1029 idReadUnit = self.__getNewId()
986 1030 else:
987 1031 idReadUnit = str(id)
988 1032
989 1033 readUnitConfObj = ReadUnitConf()
990 1034 readUnitConfObj.setup(idReadUnit, name, datatype,
991 1035 parentId=self.id, **kwargs)
992 1036
993 1037 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
994
1038
995 1039 return readUnitConfObj
996 1040
997 1041 def addProcUnit(self, inputId='0', datatype=None, name=None):
998 1042
999 idProcUnit = self.__getNewId()
1043 '''
1044 Actualizacion:
1045 Se agrego dos nuevos argumentos: topic_read (lee data de otro procUnit) y topic_write(escribe o envia data a otro procUnit)
1046 Deberia reemplazar a "inputId"
1047
1048 ** A fin de mantener el inputID, este sera la representaacion del topicoal que deben subscribirse. El ID propio de la intancia
1049 (proceso) sera el topico de la publicacion, todo sera asignado de manera dinamica.
1050
1051 '''
1052
1053 idProcUnit = self.__getNewId() #Topico para subscripcion
1000 1054
1001 1055 procUnitConfObj = ProcUnitConf()
1002 procUnitConfObj.setup(idProcUnit, name, datatype,
1003 inputId, parentId=self.id)
1056 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, #topic_read, topic_write,
1057 parentId=self.id)
1004 1058
1005 1059 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1006 1060
1007 1061 return procUnitConfObj
1008 1062
1009 1063 def removeProcUnit(self, id):
1010 1064
1011 1065 if id in list(self.procUnitConfObjDict.keys()):
1012 1066 self.procUnitConfObjDict.pop(id)
1013 1067
1014 1068 def getReadUnitId(self):
1015 1069
1016 1070 readUnitConfObj = self.getReadUnitObj()
1017 1071
1018 1072 return readUnitConfObj.id
1019 1073
1020 1074 def getReadUnitObj(self):
1021 1075
1022 1076 for obj in list(self.procUnitConfObjDict.values()):
1023 1077 if obj.getElementName() == 'ReadUnit':
1024 1078 return obj
1025 1079
1026 1080 return None
1027 1081
1028 1082 def getProcUnitObj(self, id=None, name=None):
1029 1083
1030 1084 if id != None:
1031 1085 return self.procUnitConfObjDict[id]
1032 1086
1033 1087 if name != None:
1034 1088 return self.getProcUnitObjByName(name)
1035 1089
1036 1090 return None
1037 1091
1038 1092 def getProcUnitObjByName(self, name):
1039 1093
1040 1094 for obj in list(self.procUnitConfObjDict.values()):
1041 1095 if obj.name == name:
1042 1096 return obj
1043 1097
1044 1098 return None
1045 1099
1046 1100 def procUnitItems(self):
1047 1101
1048 1102 return list(self.procUnitConfObjDict.items())
1049 1103
1050 1104 def makeXml(self):
1051 1105
1052 1106 projectElement = Element('Project')
1053 1107 projectElement.set('id', str(self.id))
1054 1108 projectElement.set('name', self.name)
1055 1109 projectElement.set('description', self.description)
1056 1110
1057 1111 for procUnitConfObj in list(self.procUnitConfObjDict.values()):
1058 1112 procUnitConfObj.makeXml(projectElement)
1059 1113
1060 1114 self.projectElement = projectElement
1061 1115
1062 1116 def writeXml(self, filename=None):
1063 1117
1064 1118 if filename == None:
1065 1119 if self.filename:
1066 1120 filename = self.filename
1067 1121 else:
1068 1122 filename = 'schain.xml'
1069 1123
1070 1124 if not filename:
1071 1125 print('filename has not been defined. Use setFilename(filename) for do it.')
1072 1126 return 0
1073 1127
1074 1128 abs_file = os.path.abspath(filename)
1075 1129
1076 1130 if not os.access(os.path.dirname(abs_file), os.W_OK):
1077 1131 print('No write permission on %s' % os.path.dirname(abs_file))
1078 1132 return 0
1079 1133
1080 1134 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1081 1135 print('File %s already exists and it could not be overwriten' % abs_file)
1082 1136 return 0
1083 1137
1084 1138 self.makeXml()
1085 1139
1086 1140 ElementTree(self.projectElement).write(abs_file, method='xml')
1087 1141
1088 1142 self.filename = abs_file
1089 1143
1090 1144 return 1
1091 1145
1092 1146 def readXml(self, filename=None):
1093 1147
1094 1148 if not filename:
1095 1149 print('filename is not defined')
1096 1150 return 0
1097 1151
1098 1152 abs_file = os.path.abspath(filename)
1099 1153
1100 1154 if not os.path.isfile(abs_file):
1101 1155 print('%s file does not exist' % abs_file)
1102 1156 return 0
1103 1157
1104 1158 self.projectElement = None
1105 1159 self.procUnitConfObjDict = {}
1106 1160
1107 1161 try:
1108 1162 self.projectElement = ElementTree().parse(abs_file)
1109 1163 except:
1110 1164 print('Error reading %s, verify file format' % filename)
1111 1165 return 0
1112 1166
1113 1167 self.project = self.projectElement.tag
1114 1168
1115 1169 self.id = self.projectElement.get('id')
1116 1170 self.name = self.projectElement.get('name')
1117 1171 self.description = self.projectElement.get('description')
1118 1172
1119 1173 readUnitElementList = self.projectElement.iter(
1120 1174 ReadUnitConf().getElementName())
1121 1175
1122 1176 for readUnitElement in readUnitElementList:
1123 1177 readUnitConfObj = ReadUnitConf()
1124 1178 readUnitConfObj.readXml(readUnitElement)
1125 1179
1126 1180 if readUnitConfObj.parentId == None:
1127 1181 readUnitConfObj.parentId = self.id
1128 1182
1129 1183 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1130 1184
1131 1185 procUnitElementList = self.projectElement.iter(
1132 1186 ProcUnitConf().getElementName())
1133 1187
1134 1188 for procUnitElement in procUnitElementList:
1135 1189 procUnitConfObj = ProcUnitConf()
1136 1190 procUnitConfObj.readXml(procUnitElement)
1137 1191
1138 1192 if procUnitConfObj.parentId == None:
1139 1193 procUnitConfObj.parentId = self.id
1140 1194
1141 1195 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1142 1196
1143 1197 self.filename = abs_file
1144 1198
1145 1199 return 1
1146 1200
1147 1201 def printattr(self):
1148 1202
1149 1203 print('Project[%s]: name = %s, description = %s' % (self.id,
1150 1204 self.name,
1151 1205 self.description))
1152 1206
1153 1207 for procUnitConfObj in list(self.procUnitConfObjDict.values()):
1154 1208 procUnitConfObj.printattr()
1155 1209
1156 1210 def createObjects(self):
1157 1211
1158 1212 for procUnitConfObj in list(self.procUnitConfObjDict.values()):
1159 procUnitConfObj.createObjects(self.plotterQueue)
1160
1161 def __connect(self, objIN, thisObj):
1162
1163 thisObj.setInput(objIN.getOutputObj())
1164
1165 def connectObjects(self):
1166
1167 for thisPUConfObj in list(self.procUnitConfObjDict.values()):
1213 print("Creating process:", procUnitConfObj.name)
1214 procUnitConfObj.createObjects(self.procUnitConfObjDict)
1215
1168 1216
1169 inputId = thisPUConfObj.getInputId()
1170
1171 if int(inputId) == 0:
1172 continue
1173
1174 # Get input object
1175 puConfINObj = self.procUnitConfObjDict[inputId]
1176 puObjIN = puConfINObj.getProcUnitObj()
1177
1178 # Get current object
1179 thisPUObj = thisPUConfObj.getProcUnitObj()
1180
1181 self.__connect(puObjIN, thisPUObj)
1217 print('All processes were created')
1182 1218
1183 1219 def __handleError(self, procUnitConfObj, modes=None, stdout=True):
1184 1220
1185 1221 import socket
1186 1222
1187 1223 if modes is None:
1188 1224 modes = self.alarm
1189 1225
1190 1226 if not self.alarm:
1191 1227 modes = []
1192 1228
1193 1229 err = traceback.format_exception(sys.exc_info()[0],
1194 1230 sys.exc_info()[1],
1195 1231 sys.exc_info()[2])
1196
1232
1197 1233 log.error('{}'.format(err[-1]), procUnitConfObj.name)
1198 1234
1199 1235 message = ''.join(err)
1200 1236
1201 1237 if stdout:
1202 1238 sys.stderr.write(message)
1203 1239
1204 1240 subject = 'SChain v%s: Error running %s\n' % (
1205 1241 schainpy.__version__, procUnitConfObj.name)
1206 1242
1207 1243 subtitle = '%s: %s\n' % (
1208 1244 procUnitConfObj.getElementName(), procUnitConfObj.name)
1209 1245 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1210 1246 socket.gethostname())
1211 1247 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1212 1248 subtitle += 'Configuration file: %s\n' % self.filename
1213 1249 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1214 1250
1215 1251 readUnitConfObj = self.getReadUnitObj()
1216 1252 if readUnitConfObj:
1217 1253 subtitle += '\nInput parameters:\n'
1218 1254 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1219 1255 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1220 1256 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1221 1257 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1222 1258 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1223 1259 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1224 1260
1225 1261 a = Alarm(
1226 1262 modes=modes,
1227 1263 email=self.email,
1228 1264 message=message,
1229 1265 subject=subject,
1230 1266 subtitle=subtitle,
1231 1267 filename=self.filename
1232 1268 )
1233 1269
1234 1270 return a
1235 1271
1236 1272 def isPaused(self):
1237 1273 return 0
1238 1274
1239 1275 def isStopped(self):
1240 1276 return 0
1241 1277
1242 1278 def runController(self):
1243 1279 '''
1244 1280 returns 0 when this process has been stopped, 1 otherwise
1245 1281 '''
1246 1282
1247 1283 if self.isPaused():
1248 1284 print('Process suspended')
1249 1285
1250 1286 while True:
1251 1287 time.sleep(0.1)
1252 1288
1253 1289 if not self.isPaused():
1254 1290 break
1255 1291
1256 1292 if self.isStopped():
1257 1293 break
1258 1294
1259 1295 print('Process reinitialized')
1260 1296
1261 1297 if self.isStopped():
1262 1298 print('Process stopped')
1263 1299 return 0
1264 1300
1265 1301 return 1
1266 1302
1267 1303 def setFilename(self, filename):
1268 1304
1269 1305 self.filename = filename
1270 1306
1271 def setPlotterQueue(self, plotter_queue):
1272
1273 raise NotImplementedError('Use schainpy.controller_api.ControllerThread instead Project class')
1274
1275 def getPlotterQueue(self):
1276
1277 raise NotImplementedError('Use schainpy.controller_api.ControllerThread instead Project class')
1278
1279 def useExternalPlotter(self):
1307 def setProxyCom(self):
1308
1309 ctx = zmq.Context()
1310 if not os.path.exists('/tmp/socketTmp'): os.mkdir('/tmp/socketTmp')
1311 xsub = ctx.socket(zmq.XSUB)
1312 xsub.bind('ipc:///tmp/socketTmp/a')
1313 xpub = ctx.socket(zmq.XPUB)
1314 xpub.bind('ipc:///tmp/socketTmp/b')
1315
1316 print("Controller Ready: Processes and proxy created")
1317 zmq.proxy(xsub, xpub)
1280 1318
1281 raise NotImplementedError('Use schainpy.controller_api.ControllerThread instead Project class')
1319
1282 1320
1283 1321 def run(self):
1284 1322
1285 1323 log.success('Starting {}'.format(self.name), tag='')
1286 1324 self.start_time = time.time()
1287 self.createObjects()
1288 self.connectObjects()
1289
1290 keyList = list(self.procUnitConfObjDict.keys())
1291 keyList.sort()
1292
1293 err = None
1294
1295 while(True):
1296
1297 is_ok = False
1325 self.createObjects()
1326 self.setProxyCom()
1298 1327
1299 for procKey in keyList:
1300
1301 procUnitConfObj = self.procUnitConfObjDict[procKey]
1302
1303 try:
1304 sts = procUnitConfObj.run()
1305 is_ok = is_ok or sts
1306 except SchainWarning:
1307 err = self.__handleError(procUnitConfObj, modes=[2, 3], stdout=False)
1308 is_ok = False
1309 break
1310 except KeyboardInterrupt:
1311 is_ok = False
1312 break
1313 except ValueError as e:
1314 time.sleep(0.5)
1315 err = self.__handleError(procUnitConfObj)
1316 is_ok = False
1317 break
1318 except:
1319 time.sleep(0.5)
1320 err = self.__handleError(procUnitConfObj)
1321 is_ok = False
1322 break
1323
1324 # If every process unit finished so end process
1325 if not(is_ok):
1326 break
1327
1328 if not self.runController():
1329 break
1328 # Iniciar todos los procesos .start(), monitoreo de procesos. ELiminar lo de abajo
1330 1329
1331 1330 # Closing every process
1332 for procKey in keyList:
1333 procUnitConfObj = self.procUnitConfObjDict[procKey]
1334 procUnitConfObj.close()
1335
1336 if err is not None:
1337 err.start()
1338 # err.join()
1339
1340 1331 log.success('{} finished (time: {}s)'.format(
1341 1332 self.name,
1342 1333 time.time()-self.start_time)) No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,657 +1,658
1 1 import os
2 2 import numpy
3 3 import time, datetime
4 4 from schainpy.model.graphics import mpldriver
5 5
6 from schainpy.model.proc.jroproc_base import Operation
6 from schainpy.model.proc.jroproc_base import MPDecorator, Operation
7
7 8
8 9 def isTimeInHourRange(datatime, xmin, xmax):
9 10
10 11 if xmin == None or xmax == None:
11 12 return 1
12 13 hour = datatime.hour + datatime.minute/60.0
13 14
14 15 if xmin < (xmax % 24):
15 16
16 17 if hour >= xmin and hour <= xmax:
17 18 return 1
18 19 else:
19 20 return 0
20 21
21 22 else:
22 23
23 24 if hour >= xmin or hour <= (xmax % 24):
24 25 return 1
25 26 else:
26 27 return 0
27 28
28 29 return 0
29 30
30 31 def isRealtime(utcdatatime):
31 32
32 33 utcnow = time.mktime(time.localtime())
33 34 delta = abs(utcnow - utcdatatime) # abs
34 35 if delta >= 30.:
35 36 return False
36 37 return True
37 38
38 39 class Figure(Operation):
39 40
40 41 __driver = mpldriver
41 42 fig = None
42 43
43 44 id = None
44 45 wintitle = None
45 46 width = None
46 47 height = None
47 48 nplots = None
48 49 timerange = None
49 50
50 51 axesObjList = []
51 52
52 53 WIDTH = 300
53 54 HEIGHT = 200
54 55 PREFIX = 'fig'
55 56
56 57 xmin = None
57 58 xmax = None
58 59
59 60 counter_imagwr = 0
60 61
61 62 figfile = None
62 63
63 64 created = False
64 65 parameters = {}
65 def __init__(self, **kwargs):
66 def __init__(self):#, **kwargs):
66 67
67 Operation.__init__(self, **kwargs)
68 Operation.__init__(self)#, **kwargs)
68 69
69 70 def __del__(self):
70 71
71 72 self.__driver.closeFigure()
72 73
73 74 def getFilename(self, name, ext='.png'):
74 75
75 76 path = '%s%03d' %(self.PREFIX, self.id)
76 77 filename = '%s_%s%s' %(self.PREFIX, name, ext)
77 78 return os.path.join(path, filename)
78 79
79 80 def getAxesObjList(self):
80 81
81 82 return self.axesObjList
82 83
83 84 def getSubplots(self):
84 85
85 86 raise NotImplementedError
86 87
87 88 def getScreenDim(self, widthplot, heightplot):
88 89
89 90 nrow, ncol = self.getSubplots()
90 91
91 92 widthscreen = widthplot*ncol
92 93 heightscreen = heightplot*nrow
93 94
94 95 return widthscreen, heightscreen
95 96
96 97 def getTimeLim(self, x, xmin=None, xmax=None, timerange=None):
97 98
98 99 # if self.xmin != None and self.xmax != None:
99 100 # if timerange == None:
100 101 # timerange = self.xmax - self.xmin
101 102 # xmin = self.xmin + timerange
102 103 # xmax = self.xmax + timerange
103 104 #
104 105 # return xmin, xmax
105 106
106 107 if timerange == None and (xmin==None or xmax==None):
107 108 timerange = 14400 #seconds
108 109
109 110 if timerange != None:
110 111 txmin = x[0] #- x[0] % min(timerange/10, 10*60)
111 112 else:
112 113 txmin = x[0] #- x[0] % 10*60
113 114
114 115 thisdatetime = datetime.datetime.utcfromtimestamp(txmin)
115 116 thisdate = datetime.datetime.combine(thisdatetime.date(), datetime.time(0,0,0))
116 117
117 118 if timerange != None:
118 119 xmin = (thisdatetime - thisdate).seconds/(60*60.)
119 120 xmax = xmin + timerange/(60*60.)
120 121
121 122 d1970 = datetime.datetime(1970,1,1)
122 123
123 124 mindt = thisdate + datetime.timedelta(hours=xmin) #- datetime.timedelta(seconds=time.timezone)
124 125 xmin_sec = (mindt - d1970).total_seconds() #time.mktime(mindt.timetuple()) - time.timezone
125 126
126 127 maxdt = thisdate + datetime.timedelta(hours=xmax) #- datetime.timedelta(seconds=time.timezone)
127 128 xmax_sec = (maxdt - d1970).total_seconds() #time.mktime(maxdt.timetuple()) - time.timezone
128 129
129 130 return xmin_sec, xmax_sec
130 131
131 132 def init(self, id, nplots, wintitle):
132 133
133 134 raise NotImplementedError("This method has been replaced by createFigure")
134 135
135 136 def createFigure(self, id, wintitle, widthplot=None, heightplot=None, show=True):
136 137
137 138 """
138 139 Crea la figura de acuerdo al driver y parametros seleccionados seleccionados.
139 140 Las dimensiones de la pantalla es calculada a partir de los atributos self.WIDTH
140 141 y self.HEIGHT y el numero de subplots (nrow, ncol)
141 142
142 143 Input:
143 144 id : Los parametros necesarios son
144 145 wintitle :
145 146
146 147 """
147 148
148 149 if widthplot == None:
149 150 widthplot = self.WIDTH
150 151
151 152 if heightplot == None:
152 153 heightplot = self.HEIGHT
153 154
154 155 self.id = id
155 156
156 157 self.wintitle = wintitle
157 158
158 159 self.widthscreen, self.heightscreen = self.getScreenDim(widthplot, heightplot)
159 160
160 161 # if self.created:
161 162 # self.__driver.closeFigure(self.fig)
162 163
163 164 if not self.created:
164 165 self.fig = self.__driver.createFigure(id=self.id,
165 166 wintitle=self.wintitle,
166 167 width=self.widthscreen,
167 168 height=self.heightscreen,
168 169 show=show)
169 170 else:
170 171 self.__driver.clearFigure(self.fig)
171 172
172 173 self.axesObjList = []
173 174 self.counter_imagwr = 0
174 175
175 176 self.created = True
176 177
177 178 def setDriver(self, driver=mpldriver):
178 179
179 180 self.__driver = driver
180 181
181 182 def setTitle(self, title):
182 183
183 184 self.__driver.setTitle(self.fig, title)
184 185
185 186 def setWinTitle(self, title):
186 187
187 188 self.__driver.setWinTitle(self.fig, title=title)
188 189
189 190 def setTextFromAxes(self, text):
190 191
191 192 raise NotImplementedError("This method has been replaced with Axes.setText")
192 193
193 194 def makeAxes(self, nrow, ncol, xpos, ypos, colspan, rowspan):
194 195
195 196 raise NotImplementedError("This method has been replaced with Axes.addAxes")
196 197
197 198 def addAxes(self, *args):
198 199 """
199 200
200 201 Input:
201 202 *args : Los parametros necesarios son
202 203 nrow, ncol, xpos, ypos, colspan, rowspan
203 204 """
204 205
205 206 axesObj = Axes(self.fig, *args)
206 207 self.axesObjList.append(axesObj)
207 208
208 209 def saveFigure(self, figpath, figfile, *args):
209 210
210 211 filename = os.path.join(figpath, figfile)
211 212
212 213 fullpath = os.path.split(filename)[0]
213 214
214 215 if not os.path.exists(fullpath):
215 216 subpath = os.path.split(fullpath)[0]
216 217
217 218 if not os.path.exists(subpath):
218 219 os.mkdir(subpath)
219 220
220 221 os.mkdir(fullpath)
221 222
222 223 self.__driver.saveFigure(self.fig, filename, *args)
223 224
224 225 def save(self, figpath, figfile=None, save=True, ftp=False, wr_period=1, thisDatetime=None, update_figfile=True):
225 226
226 227 self.counter_imagwr += 1
227 228 if self.counter_imagwr < wr_period:
228 229 return
229 230
230 231 self.counter_imagwr = 0
231 232
232 233 if save:
233 234
234 235 if not figfile:
235 236
236 237 if not thisDatetime:
237 238 raise ValueError("Saving figure: figfile or thisDatetime should be defined")
238 239 return
239 240
240 241 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
241 242 figfile = self.getFilename(name = str_datetime)
242 243
243 244 if self.figfile == None:
244 245 self.figfile = figfile
245 246
246 247 if update_figfile:
247 248 self.figfile = figfile
248 249
249 250 # store png plot to local folder
250 251 self.saveFigure(figpath, self.figfile)
251 252
252 253
253 254 if not ftp:
254 255 return
255 256
256 257 if not thisDatetime:
257 258 return
258 259
259 260 # store png plot to FTP server according to RT-Web format
260 261 ftp_filename = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS)
261 262 # ftp_filename = os.path.join(figpath, name)
262 263 self.saveFigure(figpath, ftp_filename)
263 264
264 265 def getNameToFtp(self, thisDatetime, FTP_WEI, EXP_CODE, SUB_EXP_CODE, PLOT_CODE, PLOT_POS):
265 266 YEAR_STR = '%4.4d'%thisDatetime.timetuple().tm_year
266 267 DOY_STR = '%3.3d'%thisDatetime.timetuple().tm_yday
267 268 FTP_WEI = '%2.2d'%FTP_WEI
268 269 EXP_CODE = '%3.3d'%EXP_CODE
269 270 SUB_EXP_CODE = '%2.2d'%SUB_EXP_CODE
270 271 PLOT_CODE = '%2.2d'%PLOT_CODE
271 272 PLOT_POS = '%2.2d'%PLOT_POS
272 273 name = YEAR_STR + DOY_STR + FTP_WEI + EXP_CODE + SUB_EXP_CODE + PLOT_CODE + PLOT_POS
273 274 return name
274 275
275 276 def draw(self):
276 277
277 278 self.__driver.draw(self.fig)
278 279
279 280 def run(self):
280 281
281 282 raise NotImplementedError
282 283
283 284 def close(self, show=False):
284 285
285 286 self.__driver.closeFigure(show=show, fig=self.fig)
286 287
287 288 axesList = property(getAxesObjList)
288 289
289 290
290 291 class Axes:
291 292
292 293 __driver = mpldriver
293 294 fig = None
294 295 ax = None
295 296 plot = None
296 297 __missing = 1E30
297 298 __firsttime = None
298 299
299 300 __showprofile = False
300 301
301 302 xmin = None
302 303 xmax = None
303 304 ymin = None
304 305 ymax = None
305 306 zmin = None
306 307 zmax = None
307 308
308 309 x_buffer = None
309 310 z_buffer = None
310 311
311 312 decimationx = None
312 313 decimationy = None
313 314
314 315 __MAXNUMX = 200
315 316 __MAXNUMY = 400
316 317
317 318 __MAXNUMTIME = 500
318 319
319 320 def __init__(self, *args):
320 321
321 322 """
322 323
323 324 Input:
324 325 *args : Los parametros necesarios son
325 326 fig, nrow, ncol, xpos, ypos, colspan, rowspan
326 327 """
327 328
328 329 ax = self.__driver.createAxes(*args)
329 330 self.fig = args[0]
330 331 self.ax = ax
331 332 self.plot = None
332 333
333 334 self.__firsttime = True
334 335 self.idlineList = []
335 336
336 337 self.x_buffer = numpy.array([])
337 338 self.z_buffer = numpy.array([])
338 339
339 340 def setText(self, text):
340 341
341 342 self.__driver.setAxesText(self.ax, text)
342 343
343 344 def setXAxisAsTime(self):
344 345 pass
345 346
346 347 def pline(self, x, y,
347 348 xmin=None, xmax=None,
348 349 ymin=None, ymax=None,
349 350 xlabel='', ylabel='',
350 351 title='',
351 352 **kwargs):
352 353
353 354 """
354 355
355 356 Input:
356 357 x :
357 358 y :
358 359 xmin :
359 360 xmax :
360 361 ymin :
361 362 ymax :
362 363 xlabel :
363 364 ylabel :
364 365 title :
365 366 **kwargs : Los parametros aceptados son
366 367
367 368 ticksize
368 369 ytick_visible
369 370 """
370 371
371 372 if self.__firsttime:
372 373
373 374 if xmin == None: xmin = numpy.nanmin(x)
374 375 if xmax == None: xmax = numpy.nanmax(x)
375 376 if ymin == None: ymin = numpy.nanmin(y)
376 377 if ymax == None: ymax = numpy.nanmax(y)
377 378
378 379 self.plot = self.__driver.createPline(self.ax, x, y,
379 380 xmin, xmax,
380 381 ymin, ymax,
381 382 xlabel=xlabel,
382 383 ylabel=ylabel,
383 384 title=title,
384 385 **kwargs)
385 386
386 387 self.idlineList.append(0)
387 388 self.__firsttime = False
388 389 return
389 390
390 391 self.__driver.pline(self.plot, x, y, xlabel=xlabel,
391 392 ylabel=ylabel,
392 393 title=title)
393 394
394 395 # self.__driver.pause()
395 396
396 397 def addpline(self, x, y, idline, **kwargs):
397 398 lines = self.ax.lines
398 399
399 400 if idline in self.idlineList:
400 401 self.__driver.set_linedata(self.ax, x, y, idline)
401 402
402 403 if idline not in(self.idlineList):
403 404 self.__driver.addpline(self.ax, x, y, **kwargs)
404 405 self.idlineList.append(idline)
405 406
406 407 return
407 408
408 409 def pmultiline(self, x, y,
409 410 xmin=None, xmax=None,
410 411 ymin=None, ymax=None,
411 412 xlabel='', ylabel='',
412 413 title='',
413 414 **kwargs):
414 415
415 416 if self.__firsttime:
416 417
417 418 if xmin == None: xmin = numpy.nanmin(x)
418 419 if xmax == None: xmax = numpy.nanmax(x)
419 420 if ymin == None: ymin = numpy.nanmin(y)
420 421 if ymax == None: ymax = numpy.nanmax(y)
421 422
422 423 self.plot = self.__driver.createPmultiline(self.ax, x, y,
423 424 xmin, xmax,
424 425 ymin, ymax,
425 426 xlabel=xlabel,
426 427 ylabel=ylabel,
427 428 title=title,
428 429 **kwargs)
429 430 self.__firsttime = False
430 431 return
431 432
432 433 self.__driver.pmultiline(self.plot, x, y, xlabel=xlabel,
433 434 ylabel=ylabel,
434 435 title=title)
435 436
436 437 # self.__driver.pause()
437 438
438 439 def pmultilineyaxis(self, x, y,
439 440 xmin=None, xmax=None,
440 441 ymin=None, ymax=None,
441 442 xlabel='', ylabel='',
442 443 title='',
443 444 **kwargs):
444 445
445 446 if self.__firsttime:
446 447
447 448 if xmin == None: xmin = numpy.nanmin(x)
448 449 if xmax == None: xmax = numpy.nanmax(x)
449 450 if ymin == None: ymin = numpy.nanmin(y)
450 451 if ymax == None: ymax = numpy.nanmax(y)
451 452
452 453 self.plot = self.__driver.createPmultilineYAxis(self.ax, x, y,
453 454 xmin, xmax,
454 455 ymin, ymax,
455 456 xlabel=xlabel,
456 457 ylabel=ylabel,
457 458 title=title,
458 459 **kwargs)
459 460 if self.xmin == None: self.xmin = xmin
460 461 if self.xmax == None: self.xmax = xmax
461 462 if self.ymin == None: self.ymin = ymin
462 463 if self.ymax == None: self.ymax = ymax
463 464
464 465 self.__firsttime = False
465 466 return
466 467
467 468 self.__driver.pmultilineyaxis(self.plot, x, y, xlabel=xlabel,
468 469 ylabel=ylabel,
469 470 title=title)
470 471
471 472 # self.__driver.pause()
472 473
473 474 def pcolor(self, x, y, z,
474 475 xmin=None, xmax=None,
475 476 ymin=None, ymax=None,
476 477 zmin=None, zmax=None,
477 478 xlabel='', ylabel='',
478 479 title='', colormap='jet',
479 480 **kwargs):
480 481
481 482 """
482 483 Input:
483 484 x :
484 485 y :
485 486 x :
486 487 xmin :
487 488 xmax :
488 489 ymin :
489 490 ymax :
490 491 zmin :
491 492 zmax :
492 493 xlabel :
493 494 ylabel :
494 495 title :
495 496 **kwargs : Los parametros aceptados son
496 497 ticksize=9,
497 498 cblabel=''
498 499 """
499 500
500 501 #Decimating data
501 502 xlen = len(x)
502 503 ylen = len(y)
503 504
504 505 decimationx = int(xlen/self.__MAXNUMX) + 1
505 506 decimationy = int(ylen/self.__MAXNUMY) + 1
506 507
507 508
508 509 x_buffer = x#[::decimationx]
509 510 y_buffer = y#[::decimationy]
510 511 z_buffer = z#[::decimationx, ::decimationy]
511 512 #===================================================
512 513
513 514 if self.__firsttime:
514 515
515 516 if xmin == None: xmin = numpy.nanmin(x)
516 517 if xmax == None: xmax = numpy.nanmax(x)
517 518 if ymin == None: ymin = numpy.nanmin(y)
518 519 if ymax == None: ymax = numpy.nanmax(y)
519 520 if zmin == None: zmin = numpy.nanmin(z)
520 521 if zmax == None: zmax = numpy.nanmax(z)
521 522
522 523
523 524 self.plot = self.__driver.createPcolor(self.ax, x_buffer,
524 525 y_buffer,
525 526 z_buffer,
526 527 xmin, xmax,
527 528 ymin, ymax,
528 529 zmin, zmax,
529 530 xlabel=xlabel,
530 531 ylabel=ylabel,
531 532 title=title,
532 533 colormap=colormap,
533 534 **kwargs)
534 535
535 536 if self.xmin == None: self.xmin = xmin
536 537 if self.xmax == None: self.xmax = xmax
537 538 if self.ymin == None: self.ymin = ymin
538 539 if self.ymax == None: self.ymax = ymax
539 540 if self.zmin == None: self.zmin = zmin
540 541 if self.zmax == None: self.zmax = zmax
541 542
542 543 self.__firsttime = False
543 544 return
544 545
545 546 self.__driver.pcolor(self.plot,
546 547 z_buffer,
547 548 xlabel=xlabel,
548 549 ylabel=ylabel,
549 550 title=title)
550 551
551 552 # self.__driver.pause()
552 553
553 554 def pcolorbuffer(self, x, y, z,
554 555 xmin=None, xmax=None,
555 556 ymin=None, ymax=None,
556 557 zmin=None, zmax=None,
557 558 xlabel='', ylabel='',
558 559 title='', rti = True, colormap='jet',
559 560 maxNumX = None, maxNumY = None,
560 561 **kwargs):
561 562
562 563 if maxNumX == None:
563 564 maxNumX = self.__MAXNUMTIME
564 565
565 566 if maxNumY == None:
566 567 maxNumY = self.__MAXNUMY
567 568
568 569 if self.__firsttime:
569 570 self.z_buffer = z
570 571 self.x_buffer = numpy.hstack((self.x_buffer, x))
571 572
572 573 if xmin == None: xmin = numpy.nanmin(x)
573 574 if xmax == None: xmax = numpy.nanmax(x)
574 575 if ymin == None: ymin = numpy.nanmin(y)
575 576 if ymax == None: ymax = numpy.nanmax(y)
576 577 if zmin == None: zmin = numpy.nanmin(z)
577 578 if zmax == None: zmax = numpy.nanmax(z)
578 579
579 580 self.plot = self.__driver.createPcolor(self.ax, self.x_buffer, y, z,
580 581 xmin, xmax,
581 582 ymin, ymax,
582 583 zmin, zmax,
583 584 xlabel=xlabel,
584 585 ylabel=ylabel,
585 586 title=title,
586 587 colormap=colormap,
587 588 **kwargs)
588 589
589 590 if self.xmin == None: self.xmin = xmin
590 591 if self.xmax == None: self.xmax = xmax
591 592 if self.ymin == None: self.ymin = ymin
592 593 if self.ymax == None: self.ymax = ymax
593 594 if self.zmin == None: self.zmin = zmin
594 595 if self.zmax == None: self.zmax = zmax
595 596
596 597 self.__firsttime = False
597 598 return
598 599
599 600 self.x_buffer = numpy.hstack((self.x_buffer[:-1], x[0], x[-1]))
600 601 self.z_buffer = numpy.hstack((self.z_buffer, z))
601 602 z_buffer = self.z_buffer.reshape(-1,len(y))
602 603
603 604 #Decimating data
604 605 xlen = len(self.x_buffer)
605 606 ylen = len(y)
606 607
607 608 decimationx = int(xlen/maxNumX) + 1
608 609 decimationy = int(ylen/maxNumY) + 1
609 610
610 611 x_buffer = self.x_buffer#[::decimationx]
611 612 y_buffer = y#[::decimationy]
612 613 z_buffer = z_buffer#[::decimationx, ::decimationy]
613 614 #===================================================
614 615
615 616 x_buffer, y_buffer, z_buffer = self.__fillGaps(x_buffer, y_buffer, z_buffer)
616 617
617 618 self.__driver.addpcolorbuffer(self.ax, x_buffer, y_buffer, z_buffer, self.zmin, self.zmax,
618 619 xlabel=xlabel,
619 620 ylabel=ylabel,
620 621 title=title,
621 622 colormap=colormap)
622 623
623 624 # self.__driver.pause()
624 625
625 626 def polar(self, x, y,
626 627 title='', xlabel='',ylabel='',**kwargs):
627 628
628 629 if self.__firsttime:
629 630 self.plot = self.__driver.createPolar(self.ax, x, y, title = title, xlabel = xlabel, ylabel = ylabel)
630 631 self.__firsttime = False
631 632 self.x_buffer = x
632 633 self.y_buffer = y
633 634 return
634 635
635 636 self.x_buffer = numpy.hstack((self.x_buffer,x))
636 637 self.y_buffer = numpy.hstack((self.y_buffer,y))
637 638 self.__driver.polar(self.plot, self.x_buffer, self.y_buffer, xlabel=xlabel,
638 639 ylabel=ylabel,
639 640 title=title)
640 641
641 642 # self.__driver.pause()
642 643
643 644 def __fillGaps(self, x_buffer, y_buffer, z_buffer):
644 645
645 646 if x_buffer.shape[0] < 2:
646 647 return x_buffer, y_buffer, z_buffer
647 648
648 649 deltas = x_buffer[1:] - x_buffer[0:-1]
649 650 x_median = numpy.median(deltas)
650 651
651 652 index = numpy.where(deltas > 5*x_median)
652 653
653 654 if len(index[0]) != 0:
654 655 z_buffer[index[0],::] = self.__missing
655 656 z_buffer = numpy.ma.masked_inside(z_buffer,0.99*self.__missing,1.01*self.__missing)
656 657
657 658 return x_buffer, y_buffer, z_buffer No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1542 +1,1587
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9 9
10 10 from .figure import Figure, isRealtime, isTimeInHourRange
11 11 from .plotting_codes import *
12 from schainpy.model.proc.jroproc_base import MPDecorator
12 13
14 from schainpy.utils import log
13 15
16 @MPDecorator
14 17 class SpectraPlot(Figure):
15 18
16 19 isConfig = None
17 20 __nsubplots = None
18 21
19 22 WIDTHPROF = None
20 23 HEIGHTPROF = None
21 24 PREFIX = 'spc'
22 25
23 def __init__(self, **kwargs):
24 Figure.__init__(self, **kwargs)
26 def __init__(self):#, **kwargs):
27 Figure.__init__(self)#, **kwargs)
25 28 self.isConfig = False
26 29 self.__nsubplots = 1
27
28 30 self.WIDTH = 250
29 31 self.HEIGHT = 250
30 32 self.WIDTHPROF = 120
31 33 self.HEIGHTPROF = 0
32 34 self.counter_imagwr = 0
33 35
34 36 self.PLOT_CODE = SPEC_CODE
35 37
36 38 self.FTP_WEI = None
37 39 self.EXP_CODE = None
38 40 self.SUB_EXP_CODE = None
39 41 self.PLOT_POS = None
40 42
41 43 self.__xfilter_ena = False
42 44 self.__yfilter_ena = False
43 45
44 46 def getSubplots(self):
45 47
46 48 ncol = int(numpy.sqrt(self.nplots)+0.9)
47 49 nrow = int(self.nplots*1./ncol + 0.9)
48 50
49 51 return nrow, ncol
50 52
51 53 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
52 54
53 55 self.__showprofile = showprofile
54 56 self.nplots = nplots
55 57
56 58 ncolspan = 1
57 59 colspan = 1
58 60 if showprofile:
59 61 ncolspan = 3
60 62 colspan = 2
61 63 self.__nsubplots = 2
62 64
63 65 self.createFigure(id = id,
64 66 wintitle = wintitle,
65 67 widthplot = self.WIDTH + self.WIDTHPROF,
66 68 heightplot = self.HEIGHT + self.HEIGHTPROF,
67 69 show=show)
68 70
69 71 nrow, ncol = self.getSubplots()
70 72
71 73 counter = 0
72 74 for y in range(nrow):
73 75 for x in range(ncol):
74 76
75 77 if counter >= self.nplots:
76 78 break
77 79
78 80 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
79 81
80 82 if showprofile:
81 83 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
82 84
83 85 counter += 1
84 86
85 87 def run(self, dataOut, id, wintitle="", channelList=None, showprofile=True,
86 88 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
87 89 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
88 90 server=None, folder=None, username=None, password=None,
89 91 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, realtime=False,
90 92 xaxis="frequency", colormap='jet', normFactor=None):
91 93
92 94 """
93 95
94 96 Input:
95 97 dataOut :
96 98 id :
97 99 wintitle :
98 100 channelList :
99 101 showProfile :
100 102 xmin : None,
101 103 xmax : None,
102 104 ymin : None,
103 105 ymax : None,
104 106 zmin : None,
105 107 zmax : None
106 108 """
109 if dataOut.flagNoData:
110 return dataOut
111
107 112 if realtime:
108 113 if not(isRealtime(utcdatatime = dataOut.utctime)):
109 114 print('Skipping this plot function')
110 115 return
111 116
112 117 if channelList == None:
113 118 channelIndexList = dataOut.channelIndexList
114 119 else:
115 120 channelIndexList = []
116 121 for channel in channelList:
117 122 if channel not in dataOut.channelList:
118 123 raise ValueError("Channel %d is not in dataOut.channelList" %channel)
119 124 channelIndexList.append(dataOut.channelList.index(channel))
120 125
121 126 if normFactor is None:
122 127 factor = dataOut.normFactor
123 128 else:
124 129 factor = normFactor
125 130 if xaxis == "frequency":
126 131 x = dataOut.getFreqRange(1)/1000.
127 132 xlabel = "Frequency (kHz)"
128 133
129 134 elif xaxis == "time":
130 135 x = dataOut.getAcfRange(1)
131 136 xlabel = "Time (ms)"
132 137
133 138 else:
134 139 x = dataOut.getVelRange(1)
135 140 xlabel = "Velocity (m/s)"
136 141
137 142 ylabel = "Range (Km)"
138 143
139 144 y = dataOut.getHeiRange()
140 145
141 146 z = dataOut.data_spc/factor
142 147 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
143 148 zdB = 10*numpy.log10(z)
144 149
145 150 avg = numpy.average(z, axis=1)
146 151 avgdB = 10*numpy.log10(avg)
147 152
148 153 noise = dataOut.getNoise()/factor
149 154 noisedB = 10*numpy.log10(noise)
150 155
151 156 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
152 157 title = wintitle + " Spectra"
153 158 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
154 159 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
155 160
156 161 if not self.isConfig:
157 162
158 163 nplots = len(channelIndexList)
159 164
160 165 self.setup(id=id,
161 166 nplots=nplots,
162 167 wintitle=wintitle,
163 168 showprofile=showprofile,
164 169 show=show)
165 170
166 171 if xmin == None: xmin = numpy.nanmin(x)
167 172 if xmax == None: xmax = numpy.nanmax(x)
168 173 if ymin == None: ymin = numpy.nanmin(y)
169 174 if ymax == None: ymax = numpy.nanmax(y)
170 175 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
171 176 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
172 177
173 178 self.FTP_WEI = ftp_wei
174 179 self.EXP_CODE = exp_code
175 180 self.SUB_EXP_CODE = sub_exp_code
176 181 self.PLOT_POS = plot_pos
177 182
178 183 self.isConfig = True
179 184
180 185 self.setWinTitle(title)
181 186
182 187 for i in range(self.nplots):
183 188 index = channelIndexList[i]
184 189 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
185 190 title = "Channel %d: %4.2fdB: %s" %(dataOut.channelList[index], noisedB[index], str_datetime)
186 191 if len(dataOut.beam.codeList) != 0:
187 192 title = "Ch%d:%4.2fdB,%2.2f,%2.2f:%s" %(dataOut.channelList[index], noisedB[index], dataOut.beam.azimuthList[index], dataOut.beam.zenithList[index], str_datetime)
188 193
189 194 axes = self.axesList[i*self.__nsubplots]
190 195 axes.pcolor(x, y, zdB[index,:,:],
191 196 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
192 197 xlabel=xlabel, ylabel=ylabel, title=title, colormap=colormap,
193 198 ticksize=9, cblabel='')
194 199
195 200 if self.__showprofile:
196 201 axes = self.axesList[i*self.__nsubplots +1]
197 202 axes.pline(avgdB[index,:], y,
198 203 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
199 204 xlabel='dB', ylabel='', title='',
200 205 ytick_visible=False,
201 206 grid='x')
202 207
203 208 noiseline = numpy.repeat(noisedB[index], len(y))
204 209 axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2)
205 210
206 211 self.draw()
207 212
208 213 if figfile == None:
209 214 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
210 215 name = str_datetime
211 216 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
212 217 name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith)
213 218 figfile = self.getFilename(name)
214 219
215 220 self.save(figpath=figpath,
216 221 figfile=figfile,
217 222 save=save,
218 223 ftp=ftp,
219 224 wr_period=wr_period,
220 225 thisDatetime=thisDatetime)
221 226
227 return dataOut
228 @MPDecorator
222 229 class CrossSpectraPlot(Figure):
223 230
224 231 isConfig = None
225 232 __nsubplots = None
226 233
227 234 WIDTH = None
228 235 HEIGHT = None
229 236 WIDTHPROF = None
230 237 HEIGHTPROF = None
231 238 PREFIX = 'cspc'
232 239
233 def __init__(self, **kwargs):
234 Figure.__init__(self, **kwargs)
240 def __init__(self):#, **kwargs):
241 Figure.__init__(self)#, **kwargs)
235 242 self.isConfig = False
236 243 self.__nsubplots = 4
237 244 self.counter_imagwr = 0
238 245 self.WIDTH = 250
239 246 self.HEIGHT = 250
240 247 self.WIDTHPROF = 0
241 248 self.HEIGHTPROF = 0
242 249
243 250 self.PLOT_CODE = CROSS_CODE
244 251 self.FTP_WEI = None
245 252 self.EXP_CODE = None
246 253 self.SUB_EXP_CODE = None
247 254 self.PLOT_POS = None
248 255
249 256 def getSubplots(self):
250 257
251 258 ncol = 4
252 259 nrow = self.nplots
253 260
254 261 return nrow, ncol
255 262
256 263 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
257 264
258 265 self.__showprofile = showprofile
259 266 self.nplots = nplots
260 267
261 268 ncolspan = 1
262 269 colspan = 1
263 270
264 271 self.createFigure(id = id,
265 272 wintitle = wintitle,
266 273 widthplot = self.WIDTH + self.WIDTHPROF,
267 274 heightplot = self.HEIGHT + self.HEIGHTPROF,
268 275 show=True)
269 276
270 277 nrow, ncol = self.getSubplots()
271 278
272 279 counter = 0
273 280 for y in range(nrow):
274 281 for x in range(ncol):
275 282 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
276 283
277 284 counter += 1
278 285
279 286 def run(self, dataOut, id, wintitle="", pairsList=None,
280 287 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
281 288 coh_min=None, coh_max=None, phase_min=None, phase_max=None,
282 289 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
283 290 power_cmap='jet', coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
284 291 server=None, folder=None, username=None, password=None,
285 292 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None,
286 293 xaxis='frequency'):
287 294
288 295 """
289 296
290 297 Input:
291 298 dataOut :
292 299 id :
293 300 wintitle :
294 301 channelList :
295 302 showProfile :
296 303 xmin : None,
297 304 xmax : None,
298 305 ymin : None,
299 306 ymax : None,
300 307 zmin : None,
301 308 zmax : None
302 309 """
303 310
311 if dataOut.flagNoData:
312 return dataOut
313
304 314 if pairsList == None:
305 315 pairsIndexList = dataOut.pairsIndexList
306 316 else:
307 317 pairsIndexList = []
308 318 for pair in pairsList:
309 319 if pair not in dataOut.pairsList:
310 320 raise ValueError("Pair %s is not in dataOut.pairsList" %str(pair))
311 321 pairsIndexList.append(dataOut.pairsList.index(pair))
312 322
313 323 if not pairsIndexList:
314 324 return
315 325
316 326 if len(pairsIndexList) > 4:
317 327 pairsIndexList = pairsIndexList[0:4]
318 328
319 329 if normFactor is None:
320 330 factor = dataOut.normFactor
321 331 else:
322 332 factor = normFactor
323 333 x = dataOut.getVelRange(1)
324 334 y = dataOut.getHeiRange()
325 335 z = dataOut.data_spc[:,:,:]/factor
326 336 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
327 337
328 338 noise = dataOut.noise/factor
329 339
330 340 zdB = 10*numpy.log10(z)
331 341 noisedB = 10*numpy.log10(noise)
332 342
333 343 if coh_min == None:
334 344 coh_min = 0.0
335 345 if coh_max == None:
336 346 coh_max = 1.0
337 347
338 348 if phase_min == None:
339 349 phase_min = -180
340 350 if phase_max == None:
341 351 phase_max = 180
342 352
343 353 #thisDatetime = dataOut.datatime
344 354 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
345 355 title = wintitle + " Cross-Spectra: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
346 356 # xlabel = "Velocity (m/s)"
347 357 ylabel = "Range (Km)"
348 358
349 359 if xaxis == "frequency":
350 360 x = dataOut.getFreqRange(1)/1000.
351 361 xlabel = "Frequency (kHz)"
352 362
353 363 elif xaxis == "time":
354 364 x = dataOut.getAcfRange(1)
355 365 xlabel = "Time (ms)"
356 366
357 367 else:
358 368 x = dataOut.getVelRange(1)
359 369 xlabel = "Velocity (m/s)"
360 370
361 371 if not self.isConfig:
362 372
363 373 nplots = len(pairsIndexList)
364 374
365 375 self.setup(id=id,
366 376 nplots=nplots,
367 377 wintitle=wintitle,
368 378 showprofile=False,
369 379 show=show)
370 380
371 381 avg = numpy.abs(numpy.average(z, axis=1))
372 382 avgdB = 10*numpy.log10(avg)
373 383
374 384 if xmin == None: xmin = numpy.nanmin(x)
375 385 if xmax == None: xmax = numpy.nanmax(x)
376 386 if ymin == None: ymin = numpy.nanmin(y)
377 387 if ymax == None: ymax = numpy.nanmax(y)
378 388 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
379 389 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
380 390
381 391 self.FTP_WEI = ftp_wei
382 392 self.EXP_CODE = exp_code
383 393 self.SUB_EXP_CODE = sub_exp_code
384 394 self.PLOT_POS = plot_pos
385 395
386 396 self.isConfig = True
387 397
388 398 self.setWinTitle(title)
389 399
390 400 for i in range(self.nplots):
391 401 pair = dataOut.pairsList[pairsIndexList[i]]
392 402
393 403 chan_index0 = dataOut.channelList.index(pair[0])
394 404 chan_index1 = dataOut.channelList.index(pair[1])
395 405
396 406 str_datetime = '%s %s'%(thisDatetime.strftime("%Y/%m/%d"),thisDatetime.strftime("%H:%M:%S"))
397 407 title = "Ch%d: %4.2fdB: %s" %(pair[0], noisedB[chan_index0], str_datetime)
398 408 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index0,:,:]/factor)
399 409 axes0 = self.axesList[i*self.__nsubplots]
400 410 axes0.pcolor(x, y, zdB,
401 411 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
402 412 xlabel=xlabel, ylabel=ylabel, title=title,
403 413 ticksize=9, colormap=power_cmap, cblabel='')
404 414
405 415 title = "Ch%d: %4.2fdB: %s" %(pair[1], noisedB[chan_index1], str_datetime)
406 416 zdB = 10.*numpy.log10(dataOut.data_spc[chan_index1,:,:]/factor)
407 417 axes0 = self.axesList[i*self.__nsubplots+1]
408 418 axes0.pcolor(x, y, zdB,
409 419 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
410 420 xlabel=xlabel, ylabel=ylabel, title=title,
411 421 ticksize=9, colormap=power_cmap, cblabel='')
412 422
413 423 coherenceComplex = dataOut.data_cspc[pairsIndexList[i],:,:]/numpy.sqrt(dataOut.data_spc[chan_index0,:,:]*dataOut.data_spc[chan_index1,:,:])
414 424 coherence = numpy.abs(coherenceComplex)
415 425 # phase = numpy.arctan(-1*coherenceComplex.imag/coherenceComplex.real)*180/numpy.pi
416 426 phase = numpy.arctan2(coherenceComplex.imag, coherenceComplex.real)*180/numpy.pi
417 427
418 428 title = "Coherence Ch%d * Ch%d" %(pair[0], pair[1])
419 429 axes0 = self.axesList[i*self.__nsubplots+2]
420 430 axes0.pcolor(x, y, coherence,
421 431 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=coh_min, zmax=coh_max,
422 432 xlabel=xlabel, ylabel=ylabel, title=title,
423 433 ticksize=9, colormap=coherence_cmap, cblabel='')
424 434
425 435 title = "Phase Ch%d * Ch%d" %(pair[0], pair[1])
426 436 axes0 = self.axesList[i*self.__nsubplots+3]
427 437 axes0.pcolor(x, y, phase,
428 438 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
429 439 xlabel=xlabel, ylabel=ylabel, title=title,
430 440 ticksize=9, colormap=phase_cmap, cblabel='')
431 441
432 442
433 443
434 444 self.draw()
435 445
436 446 self.save(figpath=figpath,
437 447 figfile=figfile,
438 448 save=save,
439 449 ftp=ftp,
440 450 wr_period=wr_period,
441 451 thisDatetime=thisDatetime)
442 452
453 return dataOut
443 454
455 @MPDecorator
444 456 class RTIPlot(Figure):
445 457
446 458 __isConfig = None
447 459 __nsubplots = None
448 460
449 461 WIDTHPROF = None
450 462 HEIGHTPROF = None
451 463 PREFIX = 'rti'
452 464
453 def __init__(self, **kwargs):
465 def __init__(self):#, **kwargs):
454 466
455 Figure.__init__(self, **kwargs)
467 Figure.__init__(self)#, **kwargs)
456 468 self.timerange = None
457 469 self.isConfig = False
458 470 self.__nsubplots = 1
459 471
460 472 self.WIDTH = 800
461 473 self.HEIGHT = 180
462 474 self.WIDTHPROF = 120
463 475 self.HEIGHTPROF = 0
464 476 self.counter_imagwr = 0
465 477
466 478 self.PLOT_CODE = RTI_CODE
467 479
468 480 self.FTP_WEI = None
469 481 self.EXP_CODE = None
470 482 self.SUB_EXP_CODE = None
471 483 self.PLOT_POS = None
472 484 self.tmin = None
473 485 self.tmax = None
474 486
475 487 self.xmin = None
476 488 self.xmax = None
477 489
478 490 self.figfile = None
479 491
480 492 def getSubplots(self):
481 493
482 494 ncol = 1
483 495 nrow = self.nplots
484 496
485 497 return nrow, ncol
486 498
487 499 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
488 500
489 501 self.__showprofile = showprofile
490 502 self.nplots = nplots
491 503
492 504 ncolspan = 1
493 505 colspan = 1
494 506 if showprofile:
495 507 ncolspan = 7
496 508 colspan = 6
497 509 self.__nsubplots = 2
498 510
499 511 self.createFigure(id = id,
500 512 wintitle = wintitle,
501 513 widthplot = self.WIDTH + self.WIDTHPROF,
502 514 heightplot = self.HEIGHT + self.HEIGHTPROF,
503 515 show=show)
504 516
505 517 nrow, ncol = self.getSubplots()
506 518
507 519 counter = 0
508 520 for y in range(nrow):
509 521 for x in range(ncol):
510 522
511 523 if counter >= self.nplots:
512 524 break
513 525
514 526 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
515 527
516 528 if showprofile:
517 529 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
518 530
519 531 counter += 1
520 532
521 533 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
522 534 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
523 535 timerange=None, colormap='jet',
524 536 save=False, figpath='./', lastone=0,figfile=None, ftp=False, wr_period=1, show=True,
525 537 server=None, folder=None, username=None, password=None,
526 538 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0, normFactor=None, HEIGHT=None):
527 539
528 540 """
529 541
530 542 Input:
531 543 dataOut :
532 544 id :
533 545 wintitle :
534 546 channelList :
535 547 showProfile :
536 548 xmin : None,
537 549 xmax : None,
538 550 ymin : None,
539 551 ymax : None,
540 552 zmin : None,
541 553 zmax : None
542 554 """
555 if dataOut.flagNoData:
556 return dataOut
543 557
544 558 #colormap = kwargs.get('colormap', 'jet')
545 559 if HEIGHT is not None:
546 560 self.HEIGHT = HEIGHT
547 561
548 562 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
549 563 return
550 564
551 565 if channelList == None:
552 566 channelIndexList = dataOut.channelIndexList
553 567 else:
554 568 channelIndexList = []
555 569 for channel in channelList:
556 570 if channel not in dataOut.channelList:
557 571 raise ValueError("Channel %d is not in dataOut.channelList")
558 572 channelIndexList.append(dataOut.channelList.index(channel))
559 573
560 574 if normFactor is None:
561 575 factor = dataOut.normFactor
562 576 else:
563 577 factor = normFactor
564 578
565 579 # factor = dataOut.normFactor
566 580 x = dataOut.getTimeRange()
567 581 y = dataOut.getHeiRange()
568 582
569 583 z = dataOut.data_spc/factor
570 584 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
571 585 avg = numpy.average(z, axis=1)
572 586 avgdB = 10.*numpy.log10(avg)
573 587 # avgdB = dataOut.getPower()
574 588
575 589
576 590 thisDatetime = dataOut.datatime
577 591 # thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
578 592 title = wintitle + " RTI" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
579 593 xlabel = ""
580 594 ylabel = "Range (Km)"
581 595
582 596 update_figfile = False
583 597
584 598 if self.xmax is not None and dataOut.ltctime >= self.xmax: #yong
585 599 self.counter_imagwr = wr_period
586 600 self.isConfig = False
587 601 update_figfile = True
588 602
589 603 if not self.isConfig:
590 604
591 605 nplots = len(channelIndexList)
592 606
593 607 self.setup(id=id,
594 608 nplots=nplots,
595 609 wintitle=wintitle,
596 610 showprofile=showprofile,
597 611 show=show)
598 612
599 613 if timerange != None:
600 614 self.timerange = timerange
601 615
602 616 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
603 617
604 618 noise = dataOut.noise/factor
605 619 noisedB = 10*numpy.log10(noise)
606 620
607 621 if ymin == None: ymin = numpy.nanmin(y)
608 622 if ymax == None: ymax = numpy.nanmax(y)
609 623 if zmin == None: zmin = numpy.floor(numpy.nanmin(noisedB)) - 3
610 624 if zmax == None: zmax = numpy.ceil(numpy.nanmax(avgdB)) + 3
611 625
612 626 self.FTP_WEI = ftp_wei
613 627 self.EXP_CODE = exp_code
614 628 self.SUB_EXP_CODE = sub_exp_code
615 629 self.PLOT_POS = plot_pos
616 630
617 631 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
618 632 self.isConfig = True
619 633 self.figfile = figfile
620 634 update_figfile = True
621 635
622 636 self.setWinTitle(title)
623 637
624 638 for i in range(self.nplots):
625 639 index = channelIndexList[i]
626 640 title = "Channel %d: %s" %(dataOut.channelList[index], thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
627 641 if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)):
628 642 title = title + '_' + 'azimuth,zenith=%2.2f,%2.2f'%(dataOut.azimuth, dataOut.zenith)
629 643 axes = self.axesList[i*self.__nsubplots]
630 644 zdB = avgdB[index].reshape((1,-1))
631 645 axes.pcolorbuffer(x, y, zdB,
632 646 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
633 647 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
634 648 ticksize=9, cblabel='', cbsize="1%", colormap=colormap)
635 649
636 650 if self.__showprofile:
637 651 axes = self.axesList[i*self.__nsubplots +1]
638 652 axes.pline(avgdB[index], y,
639 653 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
640 654 xlabel='dB', ylabel='', title='',
641 655 ytick_visible=False,
642 656 grid='x')
643 657
644 658 self.draw()
645 659
646 660 self.save(figpath=figpath,
647 661 figfile=figfile,
648 662 save=save,
649 663 ftp=ftp,
650 664 wr_period=wr_period,
651 665 thisDatetime=thisDatetime,
652 666 update_figfile=update_figfile)
667 return dataOut
653 668
669 @MPDecorator
654 670 class CoherenceMap(Figure):
655 671 isConfig = None
656 672 __nsubplots = None
657 673
658 674 WIDTHPROF = None
659 675 HEIGHTPROF = None
660 676 PREFIX = 'cmap'
661 677
662 def __init__(self, **kwargs):
663 Figure.__init__(self, **kwargs)
678 def __init__(self):#, **kwargs):
679 Figure.__init__(self)#, **kwargs)
664 680 self.timerange = 2*60*60
665 681 self.isConfig = False
666 682 self.__nsubplots = 1
667 683
668 684 self.WIDTH = 800
669 685 self.HEIGHT = 180
670 686 self.WIDTHPROF = 120
671 687 self.HEIGHTPROF = 0
672 688 self.counter_imagwr = 0
673 689
674 690 self.PLOT_CODE = COH_CODE
675 691
676 692 self.FTP_WEI = None
677 693 self.EXP_CODE = None
678 694 self.SUB_EXP_CODE = None
679 695 self.PLOT_POS = None
680 696 self.counter_imagwr = 0
681 697
682 698 self.xmin = None
683 699 self.xmax = None
684 700
685 701 def getSubplots(self):
686 702 ncol = 1
687 703 nrow = self.nplots*2
688 704
689 705 return nrow, ncol
690 706
691 707 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
692 708 self.__showprofile = showprofile
693 709 self.nplots = nplots
694 710
695 711 ncolspan = 1
696 712 colspan = 1
697 713 if showprofile:
698 714 ncolspan = 7
699 715 colspan = 6
700 716 self.__nsubplots = 2
701 717
702 718 self.createFigure(id = id,
703 719 wintitle = wintitle,
704 720 widthplot = self.WIDTH + self.WIDTHPROF,
705 721 heightplot = self.HEIGHT + self.HEIGHTPROF,
706 722 show=True)
707 723
708 724 nrow, ncol = self.getSubplots()
709 725
710 726 for y in range(nrow):
711 727 for x in range(ncol):
712 728
713 729 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
714 730
715 731 if showprofile:
716 732 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan+colspan, 1, 1)
717 733
718 734 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
719 735 xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None,
720 736 timerange=None, phase_min=None, phase_max=None,
721 737 save=False, figpath='./', figfile=None, ftp=False, wr_period=1,
722 738 coherence_cmap='jet', phase_cmap='RdBu_r', show=True,
723 739 server=None, folder=None, username=None, password=None,
724 740 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
725 741
742
743 if dataOut.flagNoData:
744 return dataOut
745
726 746 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
727 747 return
728 748
729 749 if pairsList == None:
730 750 pairsIndexList = dataOut.pairsIndexList
731 751 else:
732 752 pairsIndexList = []
733 753 for pair in pairsList:
734 754 if pair not in dataOut.pairsList:
735 755 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
736 756 pairsIndexList.append(dataOut.pairsList.index(pair))
737 757
738 758 if pairsIndexList == []:
739 759 return
740 760
741 761 if len(pairsIndexList) > 4:
742 762 pairsIndexList = pairsIndexList[0:4]
743 763
744 764 if phase_min == None:
745 765 phase_min = -180
746 766 if phase_max == None:
747 767 phase_max = 180
748 768
749 769 x = dataOut.getTimeRange()
750 770 y = dataOut.getHeiRange()
751 771
752 772 thisDatetime = dataOut.datatime
753 773
754 774 title = wintitle + " CoherenceMap" #: %s" %(thisDatetime.strftime("%d-%b-%Y"))
755 775 xlabel = ""
756 776 ylabel = "Range (Km)"
757 777 update_figfile = False
758 778
759 779 if not self.isConfig:
760 780 nplots = len(pairsIndexList)
761 781 self.setup(id=id,
762 782 nplots=nplots,
763 783 wintitle=wintitle,
764 784 showprofile=showprofile,
765 785 show=show)
766 786
767 787 if timerange != None:
768 788 self.timerange = timerange
769 789
770 790 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
771 791
772 792 if ymin == None: ymin = numpy.nanmin(y)
773 793 if ymax == None: ymax = numpy.nanmax(y)
774 794 if zmin == None: zmin = 0.
775 795 if zmax == None: zmax = 1.
776 796
777 797 self.FTP_WEI = ftp_wei
778 798 self.EXP_CODE = exp_code
779 799 self.SUB_EXP_CODE = sub_exp_code
780 800 self.PLOT_POS = plot_pos
781 801
782 802 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
783 803
784 804 self.isConfig = True
785 805 update_figfile = True
786 806
787 807 self.setWinTitle(title)
788 808
789 809 for i in range(self.nplots):
790 810
791 811 pair = dataOut.pairsList[pairsIndexList[i]]
792 812
793 813 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i],:,:],axis=0)
794 814 powa = numpy.average(dataOut.data_spc[pair[0],:,:],axis=0)
795 815 powb = numpy.average(dataOut.data_spc[pair[1],:,:],axis=0)
796 816
797 817
798 818 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
799 819 coherence = numpy.abs(avgcoherenceComplex)
800 820
801 821 z = coherence.reshape((1,-1))
802 822
803 823 counter = 0
804 824
805 825 title = "Coherence Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
806 826 axes = self.axesList[i*self.__nsubplots*2]
807 827 axes.pcolorbuffer(x, y, z,
808 828 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=zmin, zmax=zmax,
809 829 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
810 830 ticksize=9, cblabel='', colormap=coherence_cmap, cbsize="1%")
811 831
812 832 if self.__showprofile:
813 833 counter += 1
814 834 axes = self.axesList[i*self.__nsubplots*2 + counter]
815 835 axes.pline(coherence, y,
816 836 xmin=zmin, xmax=zmax, ymin=ymin, ymax=ymax,
817 837 xlabel='', ylabel='', title='', ticksize=7,
818 838 ytick_visible=False, nxticks=5,
819 839 grid='x')
820 840
821 841 counter += 1
822 842
823 843 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
824 844
825 845 z = phase.reshape((1,-1))
826 846
827 847 title = "Phase Ch%d * Ch%d: %s" %(pair[0], pair[1], thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
828 848 axes = self.axesList[i*self.__nsubplots*2 + counter]
829 849 axes.pcolorbuffer(x, y, z,
830 850 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax, zmin=phase_min, zmax=phase_max,
831 851 xlabel=xlabel, ylabel=ylabel, title=title, rti=True, XAxisAsTime=True,
832 852 ticksize=9, cblabel='', colormap=phase_cmap, cbsize="1%")
833 853
834 854 if self.__showprofile:
835 855 counter += 1
836 856 axes = self.axesList[i*self.__nsubplots*2 + counter]
837 857 axes.pline(phase, y,
838 858 xmin=phase_min, xmax=phase_max, ymin=ymin, ymax=ymax,
839 859 xlabel='', ylabel='', title='', ticksize=7,
840 860 ytick_visible=False, nxticks=4,
841 861 grid='x')
842 862
843 863 self.draw()
844 864
845 865 if dataOut.ltctime >= self.xmax:
846 866 self.counter_imagwr = wr_period
847 867 self.isConfig = False
848 868 update_figfile = True
849 869
850 870 self.save(figpath=figpath,
851 871 figfile=figfile,
852 872 save=save,
853 873 ftp=ftp,
854 874 wr_period=wr_period,
855 875 thisDatetime=thisDatetime,
856 876 update_figfile=update_figfile)
857 877
878 return dataOut
879
880 @MPDecorator
858 881 class PowerProfilePlot(Figure):
859 882
860 883 isConfig = None
861 884 __nsubplots = None
862 885
863 886 WIDTHPROF = None
864 887 HEIGHTPROF = None
865 888 PREFIX = 'spcprofile'
866 889
867 def __init__(self, **kwargs):
868 Figure.__init__(self, **kwargs)
890 def __init__(self):#, **kwargs):
891 Figure.__init__(self)#, **kwargs)
869 892 self.isConfig = False
870 893 self.__nsubplots = 1
871 894
872 895 self.PLOT_CODE = POWER_CODE
873 896
874 897 self.WIDTH = 300
875 898 self.HEIGHT = 500
876 899 self.counter_imagwr = 0
877 900
878 901 def getSubplots(self):
879 902 ncol = 1
880 903 nrow = 1
881 904
882 905 return nrow, ncol
883 906
884 907 def setup(self, id, nplots, wintitle, show):
885 908
886 909 self.nplots = nplots
887 910
888 911 ncolspan = 1
889 912 colspan = 1
890 913
891 914 self.createFigure(id = id,
892 915 wintitle = wintitle,
893 916 widthplot = self.WIDTH,
894 917 heightplot = self.HEIGHT,
895 918 show=show)
896 919
897 920 nrow, ncol = self.getSubplots()
898 921
899 922 counter = 0
900 923 for y in range(nrow):
901 924 for x in range(ncol):
902 925 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
903 926
904 927 def run(self, dataOut, id, wintitle="", channelList=None,
905 928 xmin=None, xmax=None, ymin=None, ymax=None,
906 929 save=False, figpath='./', figfile=None, show=True,
907 930 ftp=False, wr_period=1, server=None,
908 931 folder=None, username=None, password=None):
909 932
933 if dataOut.flagNoData:
934 return dataOut
935
910 936
911 937 if channelList == None:
912 938 channelIndexList = dataOut.channelIndexList
913 939 channelList = dataOut.channelList
914 940 else:
915 941 channelIndexList = []
916 942 for channel in channelList:
917 943 if channel not in dataOut.channelList:
918 944 raise ValueError("Channel %d is not in dataOut.channelList")
919 945 channelIndexList.append(dataOut.channelList.index(channel))
920 946
921 947 factor = dataOut.normFactor
922 948
923 949 y = dataOut.getHeiRange()
924 950
925 951 #for voltage
926 952 if dataOut.type == 'Voltage':
927 953 x = dataOut.data[channelIndexList,:] * numpy.conjugate(dataOut.data[channelIndexList,:])
928 954 x = x.real
929 955 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
930 956
931 957 #for spectra
932 958 if dataOut.type == 'Spectra':
933 959 x = dataOut.data_spc[channelIndexList,:,:]/factor
934 960 x = numpy.where(numpy.isfinite(x), x, numpy.NAN)
935 961 x = numpy.average(x, axis=1)
936 962
937 963
938 964 xdB = 10*numpy.log10(x)
939 965
940 966 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
941 967 title = wintitle + " Power Profile %s" %(thisDatetime.strftime("%d-%b-%Y"))
942 968 xlabel = "dB"
943 969 ylabel = "Range (Km)"
944 970
945 971 if not self.isConfig:
946 972
947 973 nplots = 1
948 974
949 975 self.setup(id=id,
950 976 nplots=nplots,
951 977 wintitle=wintitle,
952 978 show=show)
953 979
954 980 if ymin == None: ymin = numpy.nanmin(y)
955 981 if ymax == None: ymax = numpy.nanmax(y)
956 982 if xmin == None: xmin = numpy.nanmin(xdB)*0.9
957 983 if xmax == None: xmax = numpy.nanmax(xdB)*1.1
958 984
959 985 self.isConfig = True
960 986
961 987 self.setWinTitle(title)
962 988
963 989 title = "Power Profile: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
964 990 axes = self.axesList[0]
965 991
966 992 legendlabels = ["channel %d"%x for x in channelList]
967 993 axes.pmultiline(xdB, y,
968 994 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
969 995 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
970 996 ytick_visible=True, nxticks=5,
971 997 grid='x')
972 998
973 999 self.draw()
974 1000
975 1001 self.save(figpath=figpath,
976 1002 figfile=figfile,
977 1003 save=save,
978 1004 ftp=ftp,
979 1005 wr_period=wr_period,
980 1006 thisDatetime=thisDatetime)
1007
1008 return dataOut
981 1009
1010 @MPDecorator
982 1011 class SpectraCutPlot(Figure):
983 1012
984 1013 isConfig = None
985 1014 __nsubplots = None
986 1015
987 1016 WIDTHPROF = None
988 1017 HEIGHTPROF = None
989 1018 PREFIX = 'spc_cut'
990 1019
991 def __init__(self, **kwargs):
992 Figure.__init__(self, **kwargs)
1020 def __init__(self):#, **kwargs):
1021 Figure.__init__(self)#, **kwargs)
993 1022 self.isConfig = False
994 1023 self.__nsubplots = 1
995 1024
996 1025 self.PLOT_CODE = POWER_CODE
997 1026
998 1027 self.WIDTH = 700
999 1028 self.HEIGHT = 500
1000 1029 self.counter_imagwr = 0
1001 1030
1002 1031 def getSubplots(self):
1003 1032 ncol = 1
1004 1033 nrow = 1
1005 1034
1006 1035 return nrow, ncol
1007 1036
1008 1037 def setup(self, id, nplots, wintitle, show):
1009 1038
1010 1039 self.nplots = nplots
1011 1040
1012 1041 ncolspan = 1
1013 1042 colspan = 1
1014 1043
1015 1044 self.createFigure(id = id,
1016 1045 wintitle = wintitle,
1017 1046 widthplot = self.WIDTH,
1018 1047 heightplot = self.HEIGHT,
1019 1048 show=show)
1020 1049
1021 1050 nrow, ncol = self.getSubplots()
1022 1051
1023 1052 counter = 0
1024 1053 for y in range(nrow):
1025 1054 for x in range(ncol):
1026 1055 self.addAxes(nrow, ncol*ncolspan, y, x*ncolspan, colspan, 1)
1027 1056
1028 1057 def run(self, dataOut, id, wintitle="", channelList=None,
1029 1058 xmin=None, xmax=None, ymin=None, ymax=None,
1030 1059 save=False, figpath='./', figfile=None, show=True,
1031 1060 ftp=False, wr_period=1, server=None,
1032 1061 folder=None, username=None, password=None,
1033 1062 xaxis="frequency"):
1034 1063
1064 if dataOut.flagNoData:
1065 return dataOut
1035 1066
1036 1067 if channelList == None:
1037 1068 channelIndexList = dataOut.channelIndexList
1038 1069 channelList = dataOut.channelList
1039 1070 else:
1040 1071 channelIndexList = []
1041 1072 for channel in channelList:
1042 1073 if channel not in dataOut.channelList:
1043 1074 raise ValueError("Channel %d is not in dataOut.channelList")
1044 1075 channelIndexList.append(dataOut.channelList.index(channel))
1045 1076
1046 1077 factor = dataOut.normFactor
1047 1078
1048 1079 y = dataOut.getHeiRange()
1049 1080
1050 1081 z = dataOut.data_spc/factor
1051 1082 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
1052 1083
1053 1084 hei_index = numpy.arange(25)*3 + 20
1054 1085
1055 1086 if xaxis == "frequency":
1056 1087 x = dataOut.getFreqRange()/1000.
1057 1088 zdB = 10*numpy.log10(z[0,:,hei_index])
1058 1089 xlabel = "Frequency (kHz)"
1059 1090 ylabel = "Power (dB)"
1060 1091
1061 1092 elif xaxis == "time":
1062 1093 x = dataOut.getAcfRange()
1063 1094 zdB = z[0,:,hei_index]
1064 1095 xlabel = "Time (ms)"
1065 1096 ylabel = "ACF"
1066 1097
1067 1098 else:
1068 1099 x = dataOut.getVelRange()
1069 1100 zdB = 10*numpy.log10(z[0,:,hei_index])
1070 1101 xlabel = "Velocity (m/s)"
1071 1102 ylabel = "Power (dB)"
1072 1103
1073 1104 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
1074 1105 title = wintitle + " Range Cuts %s" %(thisDatetime.strftime("%d-%b-%Y"))
1075 1106
1076 1107 if not self.isConfig:
1077 1108
1078 1109 nplots = 1
1079 1110
1080 1111 self.setup(id=id,
1081 1112 nplots=nplots,
1082 1113 wintitle=wintitle,
1083 1114 show=show)
1084 1115
1085 1116 if xmin == None: xmin = numpy.nanmin(x)*0.9
1086 1117 if xmax == None: xmax = numpy.nanmax(x)*1.1
1087 1118 if ymin == None: ymin = numpy.nanmin(zdB)
1088 1119 if ymax == None: ymax = numpy.nanmax(zdB)
1089 1120
1090 1121 self.isConfig = True
1091 1122
1092 1123 self.setWinTitle(title)
1093 1124
1094 1125 title = "Spectra Cuts: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
1095 1126 axes = self.axesList[0]
1096 1127
1097 1128 legendlabels = ["Range = %dKm" %y[i] for i in hei_index]
1098 1129
1099 1130 axes.pmultilineyaxis( x, zdB,
1100 1131 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
1101 1132 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels,
1102 1133 ytick_visible=True, nxticks=5,
1103 1134 grid='x')
1104 1135
1105 1136 self.draw()
1106 1137
1107 1138 self.save(figpath=figpath,
1108 1139 figfile=figfile,
1109 1140 save=save,
1110 1141 ftp=ftp,
1111 1142 wr_period=wr_period,
1112 1143 thisDatetime=thisDatetime)
1113 1144
1145 return dataOut
1146
1147 @MPDecorator
1114 1148 class Noise(Figure):
1115 1149
1116 1150 isConfig = None
1117 1151 __nsubplots = None
1118 1152
1119 1153 PREFIX = 'noise'
1120 1154
1121 1155
1122 def __init__(self, **kwargs):
1123 Figure.__init__(self, **kwargs)
1156 def __init__(self):#, **kwargs):
1157 Figure.__init__(self)#, **kwargs)
1124 1158 self.timerange = 24*60*60
1125 1159 self.isConfig = False
1126 1160 self.__nsubplots = 1
1127 1161 self.counter_imagwr = 0
1128 1162 self.WIDTH = 800
1129 1163 self.HEIGHT = 400
1130 1164 self.WIDTHPROF = 120
1131 1165 self.HEIGHTPROF = 0
1132 1166 self.xdata = None
1133 1167 self.ydata = None
1134 1168
1135 1169 self.PLOT_CODE = NOISE_CODE
1136 1170
1137 1171 self.FTP_WEI = None
1138 1172 self.EXP_CODE = None
1139 1173 self.SUB_EXP_CODE = None
1140 1174 self.PLOT_POS = None
1141 1175 self.figfile = None
1142 1176
1143 1177 self.xmin = None
1144 1178 self.xmax = None
1145 1179
1146 1180 def getSubplots(self):
1147 1181
1148 1182 ncol = 1
1149 1183 nrow = 1
1150 1184
1151 1185 return nrow, ncol
1152 1186
1153 1187 def openfile(self, filename):
1154 1188 dirname = os.path.dirname(filename)
1155 1189
1156 1190 if not os.path.exists(dirname):
1157 1191 os.mkdir(dirname)
1158 1192
1159 1193 f = open(filename,'w+')
1160 1194 f.write('\n\n')
1161 1195 f.write('JICAMARCA RADIO OBSERVATORY - Noise \n')
1162 1196 f.write('DD MM YYYY HH MM SS Channel0 Channel1 Channel2 Channel3\n\n' )
1163 1197 f.close()
1164 1198
1165 1199 def save_data(self, filename_phase, data, data_datetime):
1166 1200
1167 1201 f=open(filename_phase,'a')
1168 1202
1169 1203 timetuple_data = data_datetime.timetuple()
1170 1204 day = str(timetuple_data.tm_mday)
1171 1205 month = str(timetuple_data.tm_mon)
1172 1206 year = str(timetuple_data.tm_year)
1173 1207 hour = str(timetuple_data.tm_hour)
1174 1208 minute = str(timetuple_data.tm_min)
1175 1209 second = str(timetuple_data.tm_sec)
1176 1210
1177 1211 data_msg = ''
1178 1212 for i in range(len(data)):
1179 1213 data_msg += str(data[i]) + ' '
1180 1214
1181 1215 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' ' + data_msg + '\n')
1182 1216 f.close()
1183 1217
1184 1218
1185 1219 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1186 1220
1187 1221 self.__showprofile = showprofile
1188 1222 self.nplots = nplots
1189 1223
1190 1224 ncolspan = 7
1191 1225 colspan = 6
1192 1226 self.__nsubplots = 2
1193 1227
1194 1228 self.createFigure(id = id,
1195 1229 wintitle = wintitle,
1196 1230 widthplot = self.WIDTH+self.WIDTHPROF,
1197 1231 heightplot = self.HEIGHT+self.HEIGHTPROF,
1198 1232 show=show)
1199 1233
1200 1234 nrow, ncol = self.getSubplots()
1201 1235
1202 1236 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1203 1237
1204 1238
1205 1239 def run(self, dataOut, id, wintitle="", channelList=None, showprofile='True',
1206 1240 xmin=None, xmax=None, ymin=None, ymax=None,
1207 1241 timerange=None,
1208 1242 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1209 1243 server=None, folder=None, username=None, password=None,
1210 1244 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1211 1245
1246 if dataOut.flagNoData:
1247 return dataOut
1248
1212 1249 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1213 1250 return
1214 1251
1215 1252 if channelList == None:
1216 1253 channelIndexList = dataOut.channelIndexList
1217 1254 channelList = dataOut.channelList
1218 1255 else:
1219 1256 channelIndexList = []
1220 1257 for channel in channelList:
1221 1258 if channel not in dataOut.channelList:
1222 1259 raise ValueError("Channel %d is not in dataOut.channelList")
1223 1260 channelIndexList.append(dataOut.channelList.index(channel))
1224 1261
1225 1262 x = dataOut.getTimeRange()
1226 1263 #y = dataOut.getHeiRange()
1227 1264 factor = dataOut.normFactor
1228 1265 noise = dataOut.noise[channelIndexList]/factor
1229 1266 noisedB = 10*numpy.log10(noise)
1230 1267
1231 1268 thisDatetime = dataOut.datatime
1232 1269
1233 1270 title = wintitle + " Noise" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1234 1271 xlabel = ""
1235 1272 ylabel = "Intensity (dB)"
1236 1273 update_figfile = False
1237 1274
1238 1275 if not self.isConfig:
1239 1276
1240 1277 nplots = 1
1241 1278
1242 1279 self.setup(id=id,
1243 1280 nplots=nplots,
1244 1281 wintitle=wintitle,
1245 1282 showprofile=showprofile,
1246 1283 show=show)
1247 1284
1248 1285 if timerange != None:
1249 1286 self.timerange = timerange
1250 1287
1251 1288 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1252 1289
1253 1290 if ymin == None: ymin = numpy.floor(numpy.nanmin(noisedB)) - 10.0
1254 1291 if ymax == None: ymax = numpy.nanmax(noisedB) + 10.0
1255 1292
1256 1293 self.FTP_WEI = ftp_wei
1257 1294 self.EXP_CODE = exp_code
1258 1295 self.SUB_EXP_CODE = sub_exp_code
1259 1296 self.PLOT_POS = plot_pos
1260 1297
1261 1298
1262 1299 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1263 1300 self.isConfig = True
1264 1301 self.figfile = figfile
1265 1302 self.xdata = numpy.array([])
1266 1303 self.ydata = numpy.array([])
1267 1304
1268 1305 update_figfile = True
1269 1306
1270 1307 #open file beacon phase
1271 1308 path = '%s%03d' %(self.PREFIX, self.id)
1272 1309 noise_file = os.path.join(path,'%s.txt'%self.name)
1273 1310 self.filename_noise = os.path.join(figpath,noise_file)
1274 1311
1275 1312 self.setWinTitle(title)
1276 1313
1277 1314 title = "Noise %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1278 1315
1279 1316 legendlabels = ["channel %d"%(idchannel) for idchannel in channelList]
1280 1317 axes = self.axesList[0]
1281 1318
1282 1319 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1283 1320
1284 1321 if len(self.ydata)==0:
1285 1322 self.ydata = noisedB.reshape(-1,1)
1286 1323 else:
1287 1324 self.ydata = numpy.hstack((self.ydata, noisedB.reshape(-1,1)))
1288 1325
1289 1326
1290 1327 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1291 1328 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1292 1329 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1293 1330 XAxisAsTime=True, grid='both'
1294 1331 )
1295 1332
1296 1333 self.draw()
1297 1334
1298 1335 if dataOut.ltctime >= self.xmax:
1299 1336 self.counter_imagwr = wr_period
1300 1337 self.isConfig = False
1301 1338 update_figfile = True
1302 1339
1303 1340 self.save(figpath=figpath,
1304 1341 figfile=figfile,
1305 1342 save=save,
1306 1343 ftp=ftp,
1307 1344 wr_period=wr_period,
1308 1345 thisDatetime=thisDatetime,
1309 1346 update_figfile=update_figfile)
1310 1347
1311 1348 #store data beacon phase
1312 1349 if save:
1313 1350 self.save_data(self.filename_noise, noisedB, thisDatetime)
1314 1351
1352 return dataOut
1353
1354 @MPDecorator
1315 1355 class BeaconPhase(Figure):
1316 1356
1317 1357 __isConfig = None
1318 1358 __nsubplots = None
1319 1359
1320 1360 PREFIX = 'beacon_phase'
1321 1361
1322 def __init__(self, **kwargs):
1323 Figure.__init__(self, **kwargs)
1362 def __init__(self):#, **kwargs):
1363 Figure.__init__(self)#, **kwargs)
1324 1364 self.timerange = 24*60*60
1325 1365 self.isConfig = False
1326 1366 self.__nsubplots = 1
1327 1367 self.counter_imagwr = 0
1328 1368 self.WIDTH = 800
1329 1369 self.HEIGHT = 400
1330 1370 self.WIDTHPROF = 120
1331 1371 self.HEIGHTPROF = 0
1332 1372 self.xdata = None
1333 1373 self.ydata = None
1334 1374
1335 1375 self.PLOT_CODE = BEACON_CODE
1336 1376
1337 1377 self.FTP_WEI = None
1338 1378 self.EXP_CODE = None
1339 1379 self.SUB_EXP_CODE = None
1340 1380 self.PLOT_POS = None
1341 1381
1342 1382 self.filename_phase = None
1343 1383
1344 1384 self.figfile = None
1345 1385
1346 1386 self.xmin = None
1347 1387 self.xmax = None
1348 1388
1349 1389 def getSubplots(self):
1350 1390
1351 1391 ncol = 1
1352 1392 nrow = 1
1353 1393
1354 1394 return nrow, ncol
1355 1395
1356 1396 def setup(self, id, nplots, wintitle, showprofile=True, show=True):
1357 1397
1358 1398 self.__showprofile = showprofile
1359 1399 self.nplots = nplots
1360 1400
1361 1401 ncolspan = 7
1362 1402 colspan = 6
1363 1403 self.__nsubplots = 2
1364 1404
1365 1405 self.createFigure(id = id,
1366 1406 wintitle = wintitle,
1367 1407 widthplot = self.WIDTH+self.WIDTHPROF,
1368 1408 heightplot = self.HEIGHT+self.HEIGHTPROF,
1369 1409 show=show)
1370 1410
1371 1411 nrow, ncol = self.getSubplots()
1372 1412
1373 1413 self.addAxes(nrow, ncol*ncolspan, 0, 0, colspan, 1)
1374 1414
1375 1415 def save_phase(self, filename_phase):
1376 1416 f = open(filename_phase,'w+')
1377 1417 f.write('\n\n')
1378 1418 f.write('JICAMARCA RADIO OBSERVATORY - Beacon Phase \n')
1379 1419 f.write('DD MM YYYY HH MM SS pair(2,0) pair(2,1) pair(2,3) pair(2,4)\n\n' )
1380 1420 f.close()
1381 1421
1382 1422 def save_data(self, filename_phase, data, data_datetime):
1383 1423 f=open(filename_phase,'a')
1384 1424 timetuple_data = data_datetime.timetuple()
1385 1425 day = str(timetuple_data.tm_mday)
1386 1426 month = str(timetuple_data.tm_mon)
1387 1427 year = str(timetuple_data.tm_year)
1388 1428 hour = str(timetuple_data.tm_hour)
1389 1429 minute = str(timetuple_data.tm_min)
1390 1430 second = str(timetuple_data.tm_sec)
1391 1431 f.write(day+' '+month+' '+year+' '+hour+' '+minute+' '+second+' '+str(data[0])+' '+str(data[1])+' '+str(data[2])+' '+str(data[3])+'\n')
1392 1432 f.close()
1393 1433
1394 1434
1395 1435 def run(self, dataOut, id, wintitle="", pairsList=None, showprofile='True',
1396 1436 xmin=None, xmax=None, ymin=None, ymax=None, hmin=None, hmax=None,
1397 1437 timerange=None,
1398 1438 save=False, figpath='./', figfile=None, show=True, ftp=False, wr_period=1,
1399 1439 server=None, folder=None, username=None, password=None,
1400 1440 ftp_wei=0, exp_code=0, sub_exp_code=0, plot_pos=0):
1401 1441
1442 if dataOut.flagNoData:
1443 return dataOut
1444
1402 1445 if not isTimeInHourRange(dataOut.datatime, xmin, xmax):
1403 1446 return
1404 1447
1405 1448 if pairsList == None:
1406 1449 pairsIndexList = dataOut.pairsIndexList[:10]
1407 1450 else:
1408 1451 pairsIndexList = []
1409 1452 for pair in pairsList:
1410 1453 if pair not in dataOut.pairsList:
1411 1454 raise ValueError("Pair %s is not in dataOut.pairsList" %(pair))
1412 1455 pairsIndexList.append(dataOut.pairsList.index(pair))
1413 1456
1414 1457 if pairsIndexList == []:
1415 1458 return
1416 1459
1417 1460 # if len(pairsIndexList) > 4:
1418 1461 # pairsIndexList = pairsIndexList[0:4]
1419 1462
1420 1463 hmin_index = None
1421 1464 hmax_index = None
1422 1465
1423 1466 if hmin != None and hmax != None:
1424 1467 indexes = numpy.arange(dataOut.nHeights)
1425 1468 hmin_list = indexes[dataOut.heightList >= hmin]
1426 1469 hmax_list = indexes[dataOut.heightList <= hmax]
1427 1470
1428 1471 if hmin_list.any():
1429 1472 hmin_index = hmin_list[0]
1430 1473
1431 1474 if hmax_list.any():
1432 1475 hmax_index = hmax_list[-1]+1
1433 1476
1434 1477 x = dataOut.getTimeRange()
1435 1478 #y = dataOut.getHeiRange()
1436 1479
1437 1480
1438 1481 thisDatetime = dataOut.datatime
1439 1482
1440 1483 title = wintitle + " Signal Phase" # : %s" %(thisDatetime.strftime("%d-%b-%Y"))
1441 1484 xlabel = "Local Time"
1442 1485 ylabel = "Phase (degrees)"
1443 1486
1444 1487 update_figfile = False
1445 1488
1446 1489 nplots = len(pairsIndexList)
1447 1490 #phase = numpy.zeros((len(pairsIndexList),len(dataOut.beacon_heiIndexList)))
1448 1491 phase_beacon = numpy.zeros(len(pairsIndexList))
1449 1492 for i in range(nplots):
1450 1493 pair = dataOut.pairsList[pairsIndexList[i]]
1451 1494 ccf = numpy.average(dataOut.data_cspc[pairsIndexList[i], :, hmin_index:hmax_index], axis=0)
1452 1495 powa = numpy.average(dataOut.data_spc[pair[0], :, hmin_index:hmax_index], axis=0)
1453 1496 powb = numpy.average(dataOut.data_spc[pair[1], :, hmin_index:hmax_index], axis=0)
1454 1497 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
1455 1498 phase = numpy.arctan2(avgcoherenceComplex.imag, avgcoherenceComplex.real)*180/numpy.pi
1456 1499
1457 1500 #print "Phase %d%d" %(pair[0], pair[1])
1458 1501 #print phase[dataOut.beacon_heiIndexList]
1459 1502
1460 1503 if dataOut.beacon_heiIndexList:
1461 1504 phase_beacon[i] = numpy.average(phase[dataOut.beacon_heiIndexList])
1462 1505 else:
1463 1506 phase_beacon[i] = numpy.average(phase)
1464 1507
1465 1508 if not self.isConfig:
1466 1509
1467 1510 nplots = len(pairsIndexList)
1468 1511
1469 1512 self.setup(id=id,
1470 1513 nplots=nplots,
1471 1514 wintitle=wintitle,
1472 1515 showprofile=showprofile,
1473 1516 show=show)
1474 1517
1475 1518 if timerange != None:
1476 1519 self.timerange = timerange
1477 1520
1478 1521 self.xmin, self.xmax = self.getTimeLim(x, xmin, xmax, timerange)
1479 1522
1480 1523 if ymin == None: ymin = 0
1481 1524 if ymax == None: ymax = 360
1482 1525
1483 1526 self.FTP_WEI = ftp_wei
1484 1527 self.EXP_CODE = exp_code
1485 1528 self.SUB_EXP_CODE = sub_exp_code
1486 1529 self.PLOT_POS = plot_pos
1487 1530
1488 1531 self.name = thisDatetime.strftime("%Y%m%d_%H%M%S")
1489 1532 self.isConfig = True
1490 1533 self.figfile = figfile
1491 1534 self.xdata = numpy.array([])
1492 1535 self.ydata = numpy.array([])
1493 1536
1494 1537 update_figfile = True
1495 1538
1496 1539 #open file beacon phase
1497 1540 path = '%s%03d' %(self.PREFIX, self.id)
1498 1541 beacon_file = os.path.join(path,'%s.txt'%self.name)
1499 1542 self.filename_phase = os.path.join(figpath,beacon_file)
1500 1543 #self.save_phase(self.filename_phase)
1501 1544
1502 1545
1503 1546 #store data beacon phase
1504 1547 #self.save_data(self.filename_phase, phase_beacon, thisDatetime)
1505 1548
1506 1549 self.setWinTitle(title)
1507 1550
1508 1551
1509 1552 title = "Phase Plot %s" %(thisDatetime.strftime("%Y/%m/%d %H:%M:%S"))
1510 1553
1511 1554 legendlabels = ["Pair (%d,%d)"%(pair[0], pair[1]) for pair in dataOut.pairsList]
1512 1555
1513 1556 axes = self.axesList[0]
1514 1557
1515 1558 self.xdata = numpy.hstack((self.xdata, x[0:1]))
1516 1559
1517 1560 if len(self.ydata)==0:
1518 1561 self.ydata = phase_beacon.reshape(-1,1)
1519 1562 else:
1520 1563 self.ydata = numpy.hstack((self.ydata, phase_beacon.reshape(-1,1)))
1521 1564
1522 1565
1523 1566 axes.pmultilineyaxis(x=self.xdata, y=self.ydata,
1524 1567 xmin=self.xmin, xmax=self.xmax, ymin=ymin, ymax=ymax,
1525 1568 xlabel=xlabel, ylabel=ylabel, title=title, legendlabels=legendlabels, marker='x', markersize=8, linestyle="solid",
1526 1569 XAxisAsTime=True, grid='both'
1527 1570 )
1528 1571
1529 1572 self.draw()
1530 1573
1531 1574 if dataOut.ltctime >= self.xmax:
1532 1575 self.counter_imagwr = wr_period
1533 1576 self.isConfig = False
1534 1577 update_figfile = True
1535 1578
1536 1579 self.save(figpath=figpath,
1537 1580 figfile=figfile,
1538 1581 save=save,
1539 1582 ftp=ftp,
1540 1583 wr_period=wr_period,
1541 1584 thisDatetime=thisDatetime,
1542 update_figfile=update_figfile) No newline at end of file
1585 update_figfile=update_figfile)
1586
1587 return dataOut #Yong No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,225 +1,232
1 1 '''
2 2 Created on Jul 9, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import datetime
8 8 import numpy
9
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator #YONG
10 from schainpy.utils import log
10 11 from .figure import Figure
11 12
13
14 @MPDecorator
12 15 class Scope(Figure):
13 16
14 17 isConfig = None
15 18
16 def __init__(self, **kwargs):
17 Figure.__init__(self, **kwargs)
19 def __init__(self):#, **kwargs): #YONG
20 Figure.__init__(self)#, **kwargs)
18 21 self.isConfig = False
19 22 self.WIDTH = 300
20 23 self.HEIGHT = 200
21 24 self.counter_imagwr = 0
22 25
23 26 def getSubplots(self):
24 27
25 28 nrow = self.nplots
26 29 ncol = 3
27 30 return nrow, ncol
28 31
29 32 def setup(self, id, nplots, wintitle, show):
30 33
31 34 self.nplots = nplots
32 35
33 36 self.createFigure(id=id,
34 37 wintitle=wintitle,
35 38 show=show)
36 39
37 40 nrow,ncol = self.getSubplots()
38 41 colspan = 3
39 42 rowspan = 1
40 43
41 44 for i in range(nplots):
42 45 self.addAxes(nrow, ncol, i, 0, colspan, rowspan)
43 46
44 47 def plot_iq(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
45 48 yreal = y[channelIndexList,:].real
46 49 yimag = y[channelIndexList,:].imag
47 50
48 51 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
49 52 xlabel = "Range (Km)"
50 53 ylabel = "Intensity - IQ"
51 54
52 55 if not self.isConfig:
53 56 nplots = len(channelIndexList)
54 57
55 58 self.setup(id=id,
56 59 nplots=nplots,
57 60 wintitle='',
58 61 show=show)
59 62
60 63 if xmin == None: xmin = numpy.nanmin(x)
61 64 if xmax == None: xmax = numpy.nanmax(x)
62 65 if ymin == None: ymin = min(numpy.nanmin(yreal),numpy.nanmin(yimag))
63 66 if ymax == None: ymax = max(numpy.nanmax(yreal),numpy.nanmax(yimag))
64 67
65 68 self.isConfig = True
66 69
67 70 self.setWinTitle(title)
68 71
69 72 for i in range(len(self.axesList)):
70 73 title = "Channel %d" %(i)
71 74 axes = self.axesList[i]
72 75
73 76 axes.pline(x, yreal[i,:],
74 77 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
75 78 xlabel=xlabel, ylabel=ylabel, title=title)
76 79
77 80 axes.addpline(x, yimag[i,:], idline=1, color="red", linestyle="solid", lw=2)
78 81
79 82 def plot_power(self, x, y, id, channelIndexList, thisDatetime, wintitle, show, xmin, xmax, ymin, ymax):
80 83 y = y[channelIndexList,:] * numpy.conjugate(y[channelIndexList,:])
81 84 yreal = y.real
82 85
83 86 title = wintitle + " Scope: %s" %(thisDatetime.strftime("%d-%b-%Y %H:%M:%S"))
84 87 xlabel = "Range (Km)"
85 88 ylabel = "Intensity"
86 89
87 90 if not self.isConfig:
88 91 nplots = len(channelIndexList)
89 92
90 93 self.setup(id=id,
91 94 nplots=nplots,
92 95 wintitle='',
93 96 show=show)
94 97
95 98 if xmin == None: xmin = numpy.nanmin(x)
96 99 if xmax == None: xmax = numpy.nanmax(x)
97 100 if ymin == None: ymin = numpy.nanmin(yreal)
98 101 if ymax == None: ymax = numpy.nanmax(yreal)
99 102
100 103 self.isConfig = True
101 104
102 105 self.setWinTitle(title)
103 106
104 107 for i in range(len(self.axesList)):
105 108 title = "Channel %d" %(i)
106 109 axes = self.axesList[i]
107 110 ychannel = yreal[i,:]
108 111 axes.pline(x, ychannel,
109 112 xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax,
110 113 xlabel=xlabel, ylabel=ylabel, title=title)
111 114
112 115
113 116 def run(self, dataOut, id, wintitle="", channelList=None,
114 117 xmin=None, xmax=None, ymin=None, ymax=None, save=False,
115 118 figpath='./', figfile=None, show=True, wr_period=1,
116 119 ftp=False, server=None, folder=None, username=None, password=None, type='power', **kwargs):
117 120
118 121 """
119 122
120 123 Input:
121 124 dataOut :
122 125 id :
123 126 wintitle :
124 127 channelList :
125 128 xmin : None,
126 129 xmax : None,
127 130 ymin : None,
128 131 ymax : None,
129 132 """
133 if dataOut.flagNoData:
134 return dataOut
130 135
131 136 if channelList == None:
132 137 channelIndexList = dataOut.channelIndexList
133 138 else:
134 139 channelIndexList = []
135 140 for channel in channelList:
136 141 if channel not in dataOut.channelList:
137 142 raise ValueError("Channel %d is not in dataOut.channelList")
138 143 channelIndexList.append(dataOut.channelList.index(channel))
139 144
140 145 thisDatetime = datetime.datetime.utcfromtimestamp(dataOut.getTimeRange()[0])
141 146
142 147 if dataOut.flagDataAsBlock:
143 148
144 149 for i in range(dataOut.nProfiles):
145 150
146 151 wintitle1 = wintitle + " [Profile = %d] " %i
147 152
148 153 if type == "power":
149 154 self.plot_power(dataOut.heightList,
150 155 dataOut.data[:,i,:],
151 156 id,
152 157 channelIndexList,
153 158 thisDatetime,
154 159 wintitle1,
155 160 show,
156 161 xmin,
157 162 xmax,
158 163 ymin,
159 164 ymax)
160 165
161 166 if type == "iq":
162 167 self.plot_iq(dataOut.heightList,
163 168 dataOut.data[:,i,:],
164 169 id,
165 170 channelIndexList,
166 171 thisDatetime,
167 172 wintitle1,
168 173 show,
169 174 xmin,
170 175 xmax,
171 176 ymin,
172 177 ymax)
173 178
174 179 self.draw()
175 180
176 181 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S")
177 182 figfile = self.getFilename(name = str_datetime) + "_" + str(i)
178 183
179 184 self.save(figpath=figpath,
180 185 figfile=figfile,
181 186 save=save,
182 187 ftp=ftp,
183 188 wr_period=wr_period,
184 189 thisDatetime=thisDatetime)
185 190
186 191 else:
187 192 wintitle += " [Profile = %d] " %dataOut.profileIndex
188 193
189 194 if type == "power":
190 195 self.plot_power(dataOut.heightList,
191 196 dataOut.data,
192 197 id,
193 198 channelIndexList,
194 199 thisDatetime,
195 200 wintitle,
196 201 show,
197 202 xmin,
198 203 xmax,
199 204 ymin,
200 205 ymax)
201 206
202 207 if type == "iq":
203 208 self.plot_iq(dataOut.heightList,
204 209 dataOut.data,
205 210 id,
206 211 channelIndexList,
207 212 thisDatetime,
208 213 wintitle,
209 214 show,
210 215 xmin,
211 216 xmax,
212 217 ymin,
213 218 ymax)
214 219
215 220 self.draw()
216 221
217 222 str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") + "_" + str(dataOut.profileIndex)
218 223 figfile = self.getFilename(name = str_datetime)
219 224
220 225 self.save(figpath=figpath,
221 226 figfile=figfile,
222 227 save=save,
223 228 ftp=ftp,
224 229 wr_period=wr_period,
225 thisDatetime=thisDatetime) No newline at end of file
230 thisDatetime=thisDatetime)
231
232 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1826 +1,1826
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time
14 14 import datetime
15 15 import traceback
16 16 import zmq
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 import schainpy.admin
24 23 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
25 24 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
26 25 from schainpy.utils import log
27 26 import schainpy.admin
28 27
29 28 LOCALTIME = True
30 29
31 30
32 31 def isNumber(cad):
33 32 """
34 33 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
35 34
36 35 Excepciones:
37 36 Si un determinado string no puede ser convertido a numero
38 37 Input:
39 38 str, string al cual se le analiza para determinar si convertible a un numero o no
40 39
41 40 Return:
42 41 True : si el string es uno numerico
43 42 False : no es un string numerico
44 43 """
45 44 try:
46 45 float(cad)
47 46 return True
48 47 except:
49 48 return False
50 49
51 50
52 51 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
53 52 """
54 53 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
55 54
56 55 Inputs:
57 56 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
58 57
59 58 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
60 59 segundos contados desde 01/01/1970.
61 60 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
62 61 segundos contados desde 01/01/1970.
63 62
64 63 Return:
65 64 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
66 65 fecha especificado, de lo contrario retorna False.
67 66
68 67 Excepciones:
69 68 Si el archivo no existe o no puede ser abierto
70 69 Si la cabecera no puede ser leida.
71 70
72 71 """
73 72 basicHeaderObj = BasicHeader(LOCALTIME)
74 73
75 74 try:
76 75 fp = open(filename, 'rb')
77 76 except IOError:
78 77 print("The file %s can't be opened" % (filename))
79 78 return 0
80 79
81 80 sts = basicHeaderObj.read(fp)
82 81 fp.close()
83 82
84 83 if not(sts):
85 84 print("Skipping the file %s because it has not a valid header" % (filename))
86 85 return 0
87 86
88 87 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
89 88 return 0
90 89
91 90 return 1
92 91
93 92
94 93 def isTimeInRange(thisTime, startTime, endTime):
95 94 if endTime >= startTime:
96 95 if (thisTime < startTime) or (thisTime > endTime):
97 96 return 0
98 97 return 1
99 98 else:
100 99 if (thisTime < startTime) and (thisTime > endTime):
101 100 return 0
102 101 return 1
103 102
104 103
105 104 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
106 105 """
107 106 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
108 107
109 108 Inputs:
110 109 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
111 110
112 111 startDate : fecha inicial del rango seleccionado en formato datetime.date
113 112
114 113 endDate : fecha final del rango seleccionado en formato datetime.date
115 114
116 115 startTime : tiempo inicial del rango seleccionado en formato datetime.time
117 116
118 117 endTime : tiempo final del rango seleccionado en formato datetime.time
119 118
120 119 Return:
121 120 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
122 121 fecha especificado, de lo contrario retorna False.
123 122
124 123 Excepciones:
125 124 Si el archivo no existe o no puede ser abierto
126 125 Si la cabecera no puede ser leida.
127 126
128 127 """
129 128
130 129 try:
131 130 fp = open(filename, 'rb')
132 131 except IOError:
133 132 print("The file %s can't be opened" % (filename))
134 133 return None
135 134
136 135 firstBasicHeaderObj = BasicHeader(LOCALTIME)
137 136 systemHeaderObj = SystemHeader()
138 137 radarControllerHeaderObj = RadarControllerHeader()
139 138 processingHeaderObj = ProcessingHeader()
140 139
141 140 lastBasicHeaderObj = BasicHeader(LOCALTIME)
142 141
143 142 sts = firstBasicHeaderObj.read(fp)
144 143
145 144 if not(sts):
146 145 print("[Reading] Skipping the file %s because it has not a valid header" % (filename))
147 146 return None
148 147
149 148 if not systemHeaderObj.read(fp):
150 149 return None
151 150
152 151 if not radarControllerHeaderObj.read(fp):
153 152 return None
154 153
155 154 if not processingHeaderObj.read(fp):
156 155 return None
157 156
158 157 filesize = os.path.getsize(filename)
159 158
160 159 offset = processingHeaderObj.blockSize + 24 # header size
161 160
162 161 if filesize <= offset:
163 162 print("[Reading] %s: This file has not enough data" % filename)
164 163 return None
165 164
166 165 fp.seek(-offset, 2)
167 166
168 167 sts = lastBasicHeaderObj.read(fp)
169 168
170 169 fp.close()
171 170
172 171 thisDatetime = lastBasicHeaderObj.datatime
173 172 thisTime_last_block = thisDatetime.time()
174 173
175 174 thisDatetime = firstBasicHeaderObj.datatime
176 175 thisDate = thisDatetime.date()
177 176 thisTime_first_block = thisDatetime.time()
178 177
179 178 # General case
180 179 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
181 180 #-----------o----------------------------o-----------
182 181 # startTime endTime
183 182
184 183 if endTime >= startTime:
185 184 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
186 185 return None
187 186
188 187 return thisDatetime
189 188
190 189 # If endTime < startTime then endTime belongs to the next day
191 190
192 191 #<<<<<<<<<<<o o>>>>>>>>>>>
193 192 #-----------o----------------------------o-----------
194 193 # endTime startTime
195 194
196 195 if (thisDate == startDate) and (thisTime_last_block < startTime):
197 196 return None
198 197
199 198 if (thisDate == endDate) and (thisTime_first_block > endTime):
200 199 return None
201 200
202 201 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
203 202 return None
204 203
205 204 return thisDatetime
206 205
207 206
208 207 def isFolderInDateRange(folder, startDate=None, endDate=None):
209 208 """
210 209 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
211 210
212 211 Inputs:
213 212 folder : nombre completo del directorio.
214 213 Su formato deberia ser "/path_root/?YYYYDDD"
215 214
216 215 siendo:
217 216 YYYY : Anio (ejemplo 2015)
218 217 DDD : Dia del anio (ejemplo 305)
219 218
220 219 startDate : fecha inicial del rango seleccionado en formato datetime.date
221 220
222 221 endDate : fecha final del rango seleccionado en formato datetime.date
223 222
224 223 Return:
225 224 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
226 225 fecha especificado, de lo contrario retorna False.
227 226 Excepciones:
228 227 Si el directorio no tiene el formato adecuado
229 228 """
230 229
231 230 basename = os.path.basename(folder)
232 231
233 232 if not isRadarFolder(basename):
234 233 print("The folder %s has not the rigth format" % folder)
235 234 return 0
236 235
237 236 if startDate and endDate:
238 237 thisDate = getDateFromRadarFolder(basename)
239 238
240 239 if thisDate < startDate:
241 240 return 0
242 241
243 242 if thisDate > endDate:
244 243 return 0
245 244
246 245 return 1
247 246
248 247
249 248 def isFileInDateRange(filename, startDate=None, endDate=None):
250 249 """
251 250 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
252 251
253 252 Inputs:
254 253 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
255 254
256 255 Su formato deberia ser "?YYYYDDDsss"
257 256
258 257 siendo:
259 258 YYYY : Anio (ejemplo 2015)
260 259 DDD : Dia del anio (ejemplo 305)
261 260 sss : set
262 261
263 262 startDate : fecha inicial del rango seleccionado en formato datetime.date
264 263
265 264 endDate : fecha final del rango seleccionado en formato datetime.date
266 265
267 266 Return:
268 267 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
269 268 fecha especificado, de lo contrario retorna False.
270 269 Excepciones:
271 270 Si el archivo no tiene el formato adecuado
272 271 """
273 272
274 273 basename = os.path.basename(filename)
275 274
276 275 if not isRadarFile(basename):
277 276 print("The filename %s has not the rigth format" % filename)
278 277 return 0
279 278
280 279 if startDate and endDate:
281 280 thisDate = getDateFromRadarFile(basename)
282 281
283 282 if thisDate < startDate:
284 283 return 0
285 284
286 285 if thisDate > endDate:
287 286 return 0
288 287
289 288 return 1
290 289
291 290
292 291 def getFileFromSet(path, ext, set):
293 292 validFilelist = []
294 293 fileList = os.listdir(path)
295 294
296 295 # 0 1234 567 89A BCDE
297 296 # H YYYY DDD SSS .ext
298 297
299 298 for thisFile in fileList:
300 299 try:
301 300 year = int(thisFile[1:5])
302 301 doy = int(thisFile[5:8])
303 302 except:
304 303 continue
305 304
306 305 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
307 306 continue
308 307
309 308 validFilelist.append(thisFile)
310 309
311 310 myfile = fnmatch.filter(
312 311 validFilelist, '*%4.4d%3.3d%3.3d*' % (year, doy, set))
313 312
314 313 if len(myfile) != 0:
315 314 return myfile[0]
316 315 else:
317 316 filename = '*%4.4d%3.3d%3.3d%s' % (year, doy, set, ext.lower())
318 317 print('the filename %s does not exist' % filename)
319 318 print('...going to the last file: ')
320 319
321 320 if validFilelist:
322 321 validFilelist = sorted(validFilelist, key=str.lower)
323 322 return validFilelist[-1]
324 323
325 324 return None
326 325
327 326
328 327 def getlastFileFromPath(path, ext):
329 328 """
330 329 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
331 330 al final de la depuracion devuelve el ultimo file de la lista que quedo.
332 331
333 332 Input:
334 333 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
335 334 ext : extension de los files contenidos en una carpeta
336 335
337 336 Return:
338 337 El ultimo file de una determinada carpeta, no se considera el path.
339 338 """
340 339 validFilelist = []
341 340 fileList = os.listdir(path)
342 341
343 342 # 0 1234 567 89A BCDE
344 343 # H YYYY DDD SSS .ext
345 344
346 345 for thisFile in fileList:
347 346
348 347 year = thisFile[1:5]
349 348 if not isNumber(year):
350 349 continue
351 350
352 351 doy = thisFile[5:8]
353 352 if not isNumber(doy):
354 353 continue
355 354
356 355 year = int(year)
357 356 doy = int(doy)
358 357
359 358 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
360 359 continue
361 360
362 361 validFilelist.append(thisFile)
363 362
364 363 if validFilelist:
365 364 validFilelist = sorted(validFilelist, key=str.lower)
366 365 return validFilelist[-1]
367 366
368 367 return None
369 368
370 369
371 370 def checkForRealPath(path, foldercounter, year, doy, set, ext):
372 371 """
373 372 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
374 373 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
375 374 el path exacto de un determinado file.
376 375
377 376 Example :
378 377 nombre correcto del file es .../.../D2009307/P2009307367.ext
379 378
380 379 Entonces la funcion prueba con las siguientes combinaciones
381 380 .../.../y2009307367.ext
382 381 .../.../Y2009307367.ext
383 382 .../.../x2009307/y2009307367.ext
384 383 .../.../x2009307/Y2009307367.ext
385 384 .../.../X2009307/y2009307367.ext
386 385 .../.../X2009307/Y2009307367.ext
387 386 siendo para este caso, la ultima combinacion de letras, identica al file buscado
388 387
389 388 Return:
390 389 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
391 390 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
392 391 para el filename
393 392 """
394 393 fullfilename = None
395 394 find_flag = False
396 395 filename = None
397 396
398 397 prefixDirList = [None, 'd', 'D']
399 398 if ext.lower() == ".r": # voltage
400 399 prefixFileList = ['d', 'D']
401 400 elif ext.lower() == ".pdata": # spectra
402 401 prefixFileList = ['p', 'P']
403 402 else:
404 403 return None, filename
405 404
406 405 # barrido por las combinaciones posibles
407 406 for prefixDir in prefixDirList:
408 407 thispath = path
409 408 if prefixDir != None:
410 409 # formo el nombre del directorio xYYYYDDD (x=d o x=D)
411 410 if foldercounter == 0:
412 411 thispath = os.path.join(path, "%s%04d%03d" %
413 412 (prefixDir, year, doy))
414 413 else:
415 414 thispath = os.path.join(path, "%s%04d%03d_%02d" % (
416 415 prefixDir, year, doy, foldercounter))
417 416 for prefixFile in prefixFileList: # barrido por las dos combinaciones posibles de "D"
418 417 # formo el nombre del file xYYYYDDDSSS.ext
419 418 filename = "%s%04d%03d%03d%s" % (prefixFile, year, doy, set, ext)
420 419 fullfilename = os.path.join(
421 420 thispath, filename) # formo el path completo
422 421
423 422 if os.path.exists(fullfilename): # verifico que exista
424 423 find_flag = True
425 424 break
426 425 if find_flag:
427 426 break
428 427
429 428 if not(find_flag):
430 429 return None, filename
431 430
432 431 return fullfilename, filename
433 432
434 433
435 434 def isRadarFolder(folder):
436 435 try:
437 436 year = int(folder[1:5])
438 437 doy = int(folder[5:8])
439 438 except:
440 439 return 0
441 440
442 441 return 1
443 442
444 443
445 444 def isRadarFile(file):
446 445 try:
447 446 year = int(file[1:5])
448 447 doy = int(file[5:8])
449 448 set = int(file[8:11])
450 449 except:
451 450 return 0
452 451
453 452 return 1
454 453
455 454
456 455 def getDateFromRadarFile(file):
457 456 try:
458 457 year = int(file[1:5])
459 458 doy = int(file[5:8])
460 459 set = int(file[8:11])
461 460 except:
462 461 return None
463 462
464 463 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
465 464 return thisDate
466 465
467 466
468 467 def getDateFromRadarFolder(folder):
469 468 try:
470 469 year = int(folder[1:5])
471 470 doy = int(folder[5:8])
472 471 except:
473 472 return None
474 473
475 474 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy - 1)
476 475 return thisDate
477 476
478 477
479 478 class JRODataIO:
480 479
481 480 c = 3E8
482 481
483 482 isConfig = False
484 483
485 484 basicHeaderObj = None
486 485
487 486 systemHeaderObj = None
488 487
489 488 radarControllerHeaderObj = None
490 489
491 490 processingHeaderObj = None
492 491
493 492 dtype = None
494 493
495 494 pathList = []
496 495
497 496 filenameList = []
498 497
499 498 filename = None
500 499
501 500 ext = None
502 501
503 502 flagIsNewFile = 1
504 503
505 504 flagDiscontinuousBlock = 0
506 505
507 506 flagIsNewBlock = 0
508 507
509 508 fp = None
510 509
511 510 firstHeaderSize = 0
512 511
513 512 basicHeaderSize = 24
514 513
515 514 versionFile = 1103
516 515
517 516 fileSize = None
518 517
519 518 # ippSeconds = None
520 519
521 520 fileSizeByHeader = None
522 521
523 522 fileIndex = None
524 523
525 524 profileIndex = None
526 525
527 526 blockIndex = None
528 527
529 528 nTotalBlocks = None
530 529
531 530 maxTimeStep = 30
532 531
533 532 lastUTTime = None
534 533
535 534 datablock = None
536 535
537 536 dataOut = None
538 537
539 538 blocksize = None
540 539
541 540 getByBlock = False
542 541
543 542 def __init__(self):
544 543
545 544 raise NotImplementedError
546 545
547 546 def run(self):
548 547
549 548 raise NotImplementedError
550 549
551 550 def getDtypeWidth(self):
552 551
553 552 dtype_index = get_dtype_index(self.dtype)
554 553 dtype_width = get_dtype_width(dtype_index)
555 554
556 555 return dtype_width
557 556
558 557 def getAllowedArgs(self):
559 558 if hasattr(self, '__attrs__'):
560 559 return self.__attrs__
561 560 else:
562 561 return inspect.getargspec(self.run).args
563 562
564 563
565 564 class JRODataReader(JRODataIO):
566 565
567 566 online = 0
568 567
569 568 realtime = 0
570 569
571 570 nReadBlocks = 0
572 571
573 572 delay = 10 # number of seconds waiting a new file
574 573
575 574 nTries = 3 # quantity tries
576 575
577 576 nFiles = 3 # number of files for searching
578 577
579 578 path = None
580 579
581 580 foldercounter = 0
582 581
583 582 flagNoMoreFiles = 0
584 583
585 584 datetimeList = []
586 585
587 586 __isFirstTimeOnline = 1
588 587
589 588 __printInfo = True
590 589
591 590 profileIndex = None
592 591
593 592 nTxs = 1
594 593
595 594 txIndex = None
596 595
597 596 # Added--------------------
598 597
599 598 selBlocksize = None
600 599
601 600 selBlocktime = None
602 601
603 602 def __init__(self):
604 603 """
605 604 This class is used to find data files
606 605
607 606 Example:
608 607 reader = JRODataReader()
609 608 fileList = reader.findDataFiles()
610 609
611 610 """
612 611 pass
613 612
614 613 def createObjByDefault(self):
615 614 """
616 615
617 616 """
618 617 raise NotImplementedError
619 618
620 619 def getBlockDimension(self):
621 620
622 621 raise NotImplementedError
623 622
624 623 def searchFilesOffLine(self,
625 624 path,
626 625 startDate=None,
627 626 endDate=None,
628 627 startTime=datetime.time(0, 0, 0),
629 628 endTime=datetime.time(23, 59, 59),
630 629 set=None,
631 630 expLabel='',
632 631 ext='.r',
633 632 cursor=None,
634 633 skip=None,
635 634 walk=True):
636 635
637 636 self.filenameList = []
638 637 self.datetimeList = []
639 638
640 639 pathList = []
641 640
642 641 dateList, pathList = self.findDatafiles(
643 642 path, startDate, endDate, expLabel, ext, walk, include_path=True)
644 643
645 644 if dateList == []:
646 645 return [], []
647 646
648 647 if len(dateList) > 1:
649 648 print("[Reading] Data found for date range [%s - %s]: total days = %d" % (startDate, endDate, len(dateList)))
650 649 else:
651 650 print("[Reading] Data found for date range [%s - %s]: date = %s" % (startDate, endDate, dateList[0]))
652 651
653 652 filenameList = []
654 653 datetimeList = []
655 654
656 655 for thisPath in pathList:
657 656
658 657 fileList = glob.glob1(thisPath, "*%s" % ext)
659 658 fileList.sort()
660 659
661 660 for file in fileList:
662 661
663 662 filename = os.path.join(thisPath, file)
664 663
665 664 if not isFileInDateRange(filename, startDate, endDate):
666 665 continue
667 666
668 667 thisDatetime = isFileInTimeRange(
669 668 filename, startDate, endDate, startTime, endTime)
670 669
671 670 if not(thisDatetime):
672 671 continue
673 672
674 673 filenameList.append(filename)
675 674 datetimeList.append(thisDatetime)
676 675
677 676 if cursor is not None and skip is not None:
678 677 filenameList = filenameList[cursor * skip:cursor * skip + skip]
679 678 datetimeList = datetimeList[cursor * skip:cursor * skip + skip]
680 679
681 680 if not(filenameList):
682 681 print("[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" % (startTime, endTime, ext, path))
683 682 return [], []
684 683
685 684 print("[Reading] %d file(s) was(were) found in time range: %s - %s" % (len(filenameList), startTime, endTime))
686 685
687 686 # for i in range(len(filenameList)):
688 687 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
689 688
690 689 self.filenameList = filenameList
691 690 self.datetimeList = datetimeList
692 691
693 692 return pathList, filenameList
694 693
695 694 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None):
696 695 """
697 696 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
698 697 devuelve el archivo encontrado ademas de otros datos.
699 698
700 699 Input:
701 700 path : carpeta donde estan contenidos los files que contiene data
702 701
703 702 expLabel : Nombre del subexperimento (subfolder)
704 703
705 704 ext : extension de los files
706 705
707 706 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
708 707
709 708 Return:
710 709 directory : eL directorio donde esta el file encontrado
711 710 filename : el ultimo file de una determinada carpeta
712 711 year : el anho
713 712 doy : el numero de dia del anho
714 713 set : el set del archivo
715 714
716 715
717 716 """
718 717 if not os.path.isdir(path):
719 718 return None, None, None, None, None, None
720 719
721 720 dirList = []
722 721
723 722 if not walk:
724 723 fullpath = path
725 724 foldercounter = 0
726 725 else:
727 726 # Filtra solo los directorios
728 727 for thisPath in os.listdir(path):
729 728 if not os.path.isdir(os.path.join(path, thisPath)):
730 729 continue
731 730 if not isRadarFolder(thisPath):
732 731 continue
733 732
734 733 dirList.append(thisPath)
735 734
736 735 if not(dirList):
737 736 return None, None, None, None, None, None
738 737
739 738 dirList = sorted(dirList, key=str.lower)
740 739
741 740 doypath = dirList[-1]
742 741 foldercounter = int(doypath.split('_')[1]) if len(
743 742 doypath.split('_')) > 1 else 0
744 743 fullpath = os.path.join(path, doypath, expLabel)
745 744
746 745 print("[Reading] %s folder was found: " % (fullpath))
747 746
748 747 if set == None:
749 748 filename = getlastFileFromPath(fullpath, ext)
750 749 else:
751 750 filename = getFileFromSet(fullpath, ext, set)
752 751
753 752 if not(filename):
754 753 return None, None, None, None, None, None
755 754
756 755 print("[Reading] %s file was found" % (filename))
757 756
758 757 if not(self.__verifyFile(os.path.join(fullpath, filename))):
759 758 return None, None, None, None, None, None
760 759
761 760 year = int(filename[1:5])
762 761 doy = int(filename[5:8])
763 762 set = int(filename[8:11])
764 763
765 764 return fullpath, foldercounter, filename, year, doy, set
766 765
767 766 def __setNextFileOffline(self):
768 767
769 768 idFile = self.fileIndex
770 769
771 770 while (True):
772 771 idFile += 1
773 772 if not(idFile < len(self.filenameList)):
774 773 self.flagNoMoreFiles = 1
775 774 # print "[Reading] No more Files"
776 775 return 0
777 776
778 777 filename = self.filenameList[idFile]
779 778
780 779 if not(self.__verifyFile(filename)):
781 780 continue
782 781
783 782 fileSize = os.path.getsize(filename)
784 783 fp = open(filename, 'rb')
785 784 break
786 785
787 786 self.flagIsNewFile = 1
788 787 self.fileIndex = idFile
789 788 self.filename = filename
790 789 self.fileSize = fileSize
791 790 self.fp = fp
792 791
793 792 # print "[Reading] Setting the file: %s"%self.filename
794 793
795 794 return 1
796 795
797 796 def __setNextFileOnline(self):
798 797 """
799 798 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
800 799 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
801 800 siguientes.
802 801
803 802 Affected:
804 803 self.flagIsNewFile
805 804 self.filename
806 805 self.fileSize
807 806 self.fp
808 807 self.set
809 808 self.flagNoMoreFiles
810 809
811 810 Return:
812 811 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
813 812 1 : si el file fue abierto con exito y esta listo a ser leido
814 813
815 814 Excepciones:
816 815 Si un determinado file no puede ser abierto
817 816 """
818 817 nFiles = 0
819 818 fileOk_flag = False
820 819 firstTime_flag = True
821 820
822 821 self.set += 1
823 822
824 823 if self.set > 999:
825 824 self.set = 0
826 825 self.foldercounter += 1
827 826
828 827 # busca el 1er file disponible
829 828 fullfilename, filename = checkForRealPath(
830 829 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
831 830 if fullfilename:
832 831 if self.__verifyFile(fullfilename, False):
833 832 fileOk_flag = True
834 833
835 834 # si no encuentra un file entonces espera y vuelve a buscar
836 835 if not(fileOk_flag):
837 836 # busco en los siguientes self.nFiles+1 files posibles
838 837 for nFiles in range(self.nFiles + 1):
839 838
840 839 if firstTime_flag: # si es la 1era vez entonces hace el for self.nTries veces
841 840 tries = self.nTries
842 841 else:
843 842 tries = 1 # si no es la 1era vez entonces solo lo hace una vez
844 843
845 844 for nTries in range(tries):
846 845 if firstTime_flag:
847 846 print("\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % (self.delay, filename, nTries + 1))
848 847 sleep(self.delay)
849 848 else:
850 849 print("\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext))
851 850
852 851 fullfilename, filename = checkForRealPath(
853 852 self.path, self.foldercounter, self.year, self.doy, self.set, self.ext)
854 853 if fullfilename:
855 854 if self.__verifyFile(fullfilename):
856 855 fileOk_flag = True
857 856 break
858 857
859 858 if fileOk_flag:
860 859 break
861 860
862 861 firstTime_flag = False
863 862
864 863 log.warning('Skipping the file {} due to this file doesn\'t exist'.format(filename))
865 864 self.set += 1
866 865
867 866 # si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
868 867 if nFiles == (self.nFiles - 1):
869 868 self.set = 0
870 869 self.doy += 1
871 870 self.foldercounter = 0
872 871
873 872 if fileOk_flag:
874 873 self.fileSize = os.path.getsize(fullfilename)
875 874 self.filename = fullfilename
876 875 self.flagIsNewFile = 1
877 876 if self.fp != None:
878 877 self.fp.close()
879 878 self.fp = open(fullfilename, 'rb')
880 879 self.flagNoMoreFiles = 0
881 880 # print '[Reading] Setting the file: %s' % fullfilename
882 881 else:
883 882 self.fileSize = 0
884 883 self.filename = None
885 884 self.flagIsNewFile = 0
886 885 self.fp = None
887 886 self.flagNoMoreFiles = 1
887 # print '[Reading] No more files to read'
888 888
889 889 return fileOk_flag
890 890
891 891 def setNextFile(self):
892 892 if self.fp != None:
893 893 self.fp.close()
894 894
895 895 if self.online:
896 896 newFile = self.__setNextFileOnline()
897 897 else:
898 898 newFile = self.__setNextFileOffline()
899 899
900 if not(newFile):
901 raise schainpy.admin.SchainWarning('No more files to read')
900 if not(newFile):
901 raise(schainpy.admin.SchainWarning('No more files to read'))
902 902 return 0
903 903
904 904 if self.verbose:
905 905 print('[Reading] Setting the file: %s' % self.filename)
906 906
907 907 self.__readFirstHeader()
908 908 self.nReadBlocks = 0
909 909 return 1
910 910
911 911 def __waitNewBlock(self):
912 912 """
913 913 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
914 914
915 915 Si el modo de lectura es OffLine siempre retorn 0
916 916 """
917 917 if not self.online:
918 918 return 0
919 919
920 920 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
921 921 return 0
922 922
923 923 currentPointer = self.fp.tell()
924 924
925 925 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
926 926
927 927 for nTries in range(self.nTries):
928 928
929 929 self.fp.close()
930 930 self.fp = open(self.filename, 'rb')
931 931 self.fp.seek(currentPointer)
932 932
933 933 self.fileSize = os.path.getsize(self.filename)
934 934 currentSize = self.fileSize - currentPointer
935 935
936 936 if (currentSize >= neededSize):
937 937 self.basicHeaderObj.read(self.fp)
938 938 return 1
939 939
940 940 if self.fileSize == self.fileSizeByHeader:
941 941 # self.flagEoF = True
942 942 return 0
943 943
944 944 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
945 945 sleep(self.delay)
946 946
947 947 return 0
948 948
949 949 def waitDataBlock(self, pointer_location):
950 950
951 951 currentPointer = pointer_location
952 952
953 953 neededSize = self.processingHeaderObj.blockSize # + self.basicHeaderSize
954 954
955 955 for nTries in range(self.nTries):
956 956 self.fp.close()
957 957 self.fp = open(self.filename, 'rb')
958 958 self.fp.seek(currentPointer)
959 959
960 960 self.fileSize = os.path.getsize(self.filename)
961 961 currentSize = self.fileSize - currentPointer
962 962
963 963 if (currentSize >= neededSize):
964 964 return 1
965 965
966 966 print("[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries + 1))
967 967 sleep(self.delay)
968 968
969 969 return 0
970 970
971 971 def __jumpToLastBlock(self):
972 972
973 973 if not(self.__isFirstTimeOnline):
974 974 return
975 975
976 976 csize = self.fileSize - self.fp.tell()
977 977 blocksize = self.processingHeaderObj.blockSize
978 978
979 979 # salta el primer bloque de datos
980 980 if csize > self.processingHeaderObj.blockSize:
981 981 self.fp.seek(self.fp.tell() + blocksize)
982 982 else:
983 983 return
984 984
985 985 csize = self.fileSize - self.fp.tell()
986 986 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
987 987 while True:
988 988
989 989 if self.fp.tell() < self.fileSize:
990 990 self.fp.seek(self.fp.tell() + neededsize)
991 991 else:
992 992 self.fp.seek(self.fp.tell() - neededsize)
993 993 break
994 994
995 995 # csize = self.fileSize - self.fp.tell()
996 996 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
997 997 # factor = int(csize/neededsize)
998 998 # if factor > 0:
999 999 # self.fp.seek(self.fp.tell() + factor*neededsize)
1000 1000
1001 1001 self.flagIsNewFile = 0
1002 1002 self.__isFirstTimeOnline = 0
1003 1003
1004 1004 def __setNewBlock(self):
1005 1005 # if self.server is None:
1006 1006 if self.fp == None:
1007 1007 return 0
1008 1008
1009 1009 # if self.online:
1010 1010 # self.__jumpToLastBlock()
1011 1011
1012 1012 if self.flagIsNewFile:
1013 1013 self.lastUTTime = self.basicHeaderObj.utc
1014 1014 return 1
1015 1015
1016 1016 if self.realtime:
1017 1017 self.flagDiscontinuousBlock = 1
1018 1018 if not(self.setNextFile()):
1019 1019 return 0
1020 1020 else:
1021 1021 return 1
1022 1022 # if self.server is None:
1023 1023 currentSize = self.fileSize - self.fp.tell()
1024 1024 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1025 1025 if (currentSize >= neededSize):
1026 1026 self.basicHeaderObj.read(self.fp)
1027 1027 self.lastUTTime = self.basicHeaderObj.utc
1028 1028 return 1
1029 1029 # else:
1030 1030 # self.basicHeaderObj.read(self.zHeader)
1031 1031 # self.lastUTTime = self.basicHeaderObj.utc
1032 1032 # return 1
1033 1033 if self.__waitNewBlock():
1034 1034 self.lastUTTime = self.basicHeaderObj.utc
1035 1035 return 1
1036 1036 # if self.server is None:
1037 1037 if not(self.setNextFile()):
1038 1038 return 0
1039 1039
1040 1040 deltaTime = self.basicHeaderObj.utc - self.lastUTTime
1041 1041 self.lastUTTime = self.basicHeaderObj.utc
1042 1042
1043 1043 self.flagDiscontinuousBlock = 0
1044 1044
1045 1045 if deltaTime > self.maxTimeStep:
1046 1046 self.flagDiscontinuousBlock = 1
1047 1047
1048 1048 return 1
1049 1049
1050 1050 def readNextBlock(self):
1051 1051
1052 1052 # Skip block out of startTime and endTime
1053 1053 while True:
1054 1054 if not(self.__setNewBlock()):
1055 raise schainpy
1055 raise(schainpy.admin.SchainWarning('No more files'))
1056 1056 return 0
1057 1057
1058 1058 if not(self.readBlock()):
1059 1059 return 0
1060 1060
1061 1061 self.getBasicHeader()
1062 1062 if (self.dataOut.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or (self.dataOut.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
1063 1063 print("[Reading] Block No. %d/%d -> %s [Skipping]" % (self.nReadBlocks,
1064 1064 self.processingHeaderObj.dataBlocksPerFile,
1065 1065 self.dataOut.datatime.ctime()))
1066 1066 continue
1067 1067
1068 1068 break
1069 1069
1070 1070 if self.verbose:
1071 1071 print("[Reading] Block No. %d/%d -> %s" % (self.nReadBlocks,
1072 1072 self.processingHeaderObj.dataBlocksPerFile,
1073 1073 self.dataOut.datatime.ctime()))
1074 1074 return 1
1075 1075
1076 1076 def __readFirstHeader(self):
1077 1077
1078 1078 self.basicHeaderObj.read(self.fp)
1079 1079 self.systemHeaderObj.read(self.fp)
1080 1080 self.radarControllerHeaderObj.read(self.fp)
1081 1081 self.processingHeaderObj.read(self.fp)
1082 1082
1083 1083 self.firstHeaderSize = self.basicHeaderObj.size
1084 1084
1085 1085 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
1086 1086 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
1087 1087 if datatype == 0:
1088 1088 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
1089 1089 elif datatype == 1:
1090 1090 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
1091 1091 elif datatype == 2:
1092 1092 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
1093 1093 elif datatype == 3:
1094 1094 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
1095 1095 elif datatype == 4:
1096 1096 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
1097 1097 elif datatype == 5:
1098 1098 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
1099 1099 else:
1100 1100 raise ValueError('Data type was not defined')
1101 1101
1102 1102 self.dtype = datatype_str
1103 1103 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1104 1104 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
1105 1105 self.firstHeaderSize + self.basicHeaderSize * \
1106 1106 (self.processingHeaderObj.dataBlocksPerFile - 1)
1107 1107 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1108 1108 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1109 1109 self.getBlockDimension()
1110 1110
1111 1111 def __verifyFile(self, filename, msgFlag=True):
1112 1112
1113 1113 msg = None
1114 1114
1115 1115 try:
1116 1116 fp = open(filename, 'rb')
1117 1117 except IOError:
1118 1118
1119 1119 if msgFlag:
1120 1120 print("[Reading] File %s can't be opened" % (filename))
1121 1121
1122 1122 return False
1123 1123
1124 1124 currentPosition = fp.tell()
1125 1125 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1126 1126
1127 1127 if neededSize == 0:
1128 1128 basicHeaderObj = BasicHeader(LOCALTIME)
1129 1129 systemHeaderObj = SystemHeader()
1130 1130 radarControllerHeaderObj = RadarControllerHeader()
1131 1131 processingHeaderObj = ProcessingHeader()
1132 1132
1133 1133 if not(basicHeaderObj.read(fp)):
1134 1134 fp.close()
1135 1135 return False
1136 1136
1137 1137 if not(systemHeaderObj.read(fp)):
1138 1138 fp.close()
1139 1139 return False
1140 1140
1141 1141 if not(radarControllerHeaderObj.read(fp)):
1142 1142 fp.close()
1143 1143 return False
1144 1144
1145 1145 if not(processingHeaderObj.read(fp)):
1146 1146 fp.close()
1147 1147 return False
1148 1148
1149 1149 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1150 1150 else:
1151 1151 msg = "[Reading] Skipping the file %s due to it hasn't enough data" % filename
1152 1152
1153 1153 fp.close()
1154 1154
1155 1155 fileSize = os.path.getsize(filename)
1156 1156 currentSize = fileSize - currentPosition
1157 1157
1158 1158 if currentSize < neededSize:
1159 1159 if msgFlag and (msg != None):
1160 1160 print(msg)
1161 1161 return False
1162 1162
1163 1163 return True
1164 1164
1165 1165 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1166 1166
1167 1167 path_empty = True
1168 1168
1169 1169 dateList = []
1170 1170 pathList = []
1171 1171
1172 1172 multi_path = path.split(',')
1173 1173
1174 1174 if not walk:
1175 1175
1176 1176 for single_path in multi_path:
1177 1177
1178 1178 if not os.path.isdir(single_path):
1179 1179 continue
1180 1180
1181 1181 fileList = glob.glob1(single_path, "*" + ext)
1182 1182
1183 1183 if not fileList:
1184 1184 continue
1185 1185
1186 1186 path_empty = False
1187 1187
1188 1188 fileList.sort()
1189 1189
1190 1190 for thisFile in fileList:
1191 1191
1192 1192 if not os.path.isfile(os.path.join(single_path, thisFile)):
1193 1193 continue
1194 1194
1195 1195 if not isRadarFile(thisFile):
1196 1196 continue
1197 1197
1198 1198 if not isFileInDateRange(thisFile, startDate, endDate):
1199 1199 continue
1200 1200
1201 1201 thisDate = getDateFromRadarFile(thisFile)
1202 1202
1203 1203 if thisDate in dateList:
1204 1204 continue
1205 1205
1206 1206 dateList.append(thisDate)
1207 1207 pathList.append(single_path)
1208 1208
1209 1209 else:
1210 1210 for single_path in multi_path:
1211 1211
1212 1212 if not os.path.isdir(single_path):
1213 1213 continue
1214 1214
1215 1215 dirList = []
1216 1216
1217 1217 for thisPath in os.listdir(single_path):
1218 1218
1219 1219 if not os.path.isdir(os.path.join(single_path, thisPath)):
1220 1220 continue
1221 1221
1222 1222 if not isRadarFolder(thisPath):
1223 1223 continue
1224 1224
1225 1225 if not isFolderInDateRange(thisPath, startDate, endDate):
1226 1226 continue
1227 1227
1228 1228 dirList.append(thisPath)
1229 1229
1230 1230 if not dirList:
1231 1231 continue
1232 1232
1233 1233 dirList.sort()
1234 1234
1235 1235 for thisDir in dirList:
1236 1236
1237 1237 datapath = os.path.join(single_path, thisDir, expLabel)
1238 1238 fileList = glob.glob1(datapath, "*" + ext)
1239 1239
1240 1240 if not fileList:
1241 1241 continue
1242 1242
1243 1243 path_empty = False
1244 1244
1245 1245 thisDate = getDateFromRadarFolder(thisDir)
1246 1246
1247 1247 pathList.append(datapath)
1248 1248 dateList.append(thisDate)
1249 1249
1250 1250 dateList.sort()
1251 1251
1252 1252 if walk:
1253 1253 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1254 1254 else:
1255 1255 pattern_path = multi_path[0]
1256 1256
1257 1257 if path_empty:
1258 1258 print("[Reading] No *%s files in %s for %s to %s" % (ext, pattern_path, startDate, endDate))
1259 1259 else:
1260 1260 if not dateList:
1261 1261 print("[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" % (startDate, endDate, ext, path))
1262 1262
1263 1263 if include_path:
1264 1264 return dateList, pathList
1265 1265
1266 1266 return dateList
1267 1267
1268 1268 def setup(self,
1269 1269 path=None,
1270 1270 startDate=None,
1271 1271 endDate=None,
1272 1272 startTime=datetime.time(0, 0, 0),
1273 1273 endTime=datetime.time(23, 59, 59),
1274 1274 set=None,
1275 1275 expLabel="",
1276 1276 ext=None,
1277 1277 online=False,
1278 1278 delay=60,
1279 1279 walk=True,
1280 1280 getblock=False,
1281 1281 nTxs=1,
1282 1282 realtime=False,
1283 1283 blocksize=None,
1284 1284 blocktime=None,
1285 1285 skip=None,
1286 1286 cursor=None,
1287 1287 warnings=True,
1288 1288 verbose=True,
1289 1289 server=None,
1290 1290 format=None,
1291 1291 oneDDict=None,
1292 1292 twoDDict=None,
1293 1293 ind2DList=None):
1294 1294 if server is not None:
1295 1295 if 'tcp://' in server:
1296 1296 address = server
1297 1297 else:
1298 1298 address = 'ipc:///tmp/%s' % server
1299 1299 self.server = address
1300 1300 self.context = zmq.Context()
1301 1301 self.receiver = self.context.socket(zmq.PULL)
1302 1302 self.receiver.connect(self.server)
1303 1303 time.sleep(0.5)
1304 1304 print('[Starting] ReceiverData from {}'.format(self.server))
1305 1305 else:
1306 1306 self.server = None
1307 1307 if path == None:
1308 1308 raise ValueError("[Reading] The path is not valid")
1309 1309
1310 1310 if ext == None:
1311 1311 ext = self.ext
1312 1312
1313 1313 if online:
1314 1314 print("[Reading] Searching files in online mode...")
1315 1315
1316 1316 for nTries in range(self.nTries):
1317 1317 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(
1318 1318 path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1319 1319
1320 1320 if fullpath:
1321 1321 break
1322 1322
1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (delay, path, nTries + 1))
1324 sleep(delay)
1323 print('[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries + 1))
1324 sleep(self.delay)
1325 1325
1326 if not(fullpath):
1327 raise schainpy.admin.SchainWarning('There isn\'t any valid file in {}'.format(path))
1326 if not(fullpath):
1327 raise(schainpy.admin.SchainWarning('There isn\'t any valid file in {}'.format(path)))
1328 1328 return
1329 1329
1330 1330 self.year = year
1331 1331 self.doy = doy
1332 1332 self.set = set - 1
1333 1333 self.path = path
1334 1334 self.foldercounter = foldercounter
1335 1335 last_set = None
1336 1336 else:
1337 1337 print("[Reading] Searching files in offline mode ...")
1338 1338 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1339 1339 startTime=startTime, endTime=endTime,
1340 1340 set=set, expLabel=expLabel, ext=ext,
1341 1341 walk=walk, cursor=cursor,
1342 1342 skip=skip)
1343 1343
1344 1344 if not(pathList):
1345 1345 self.fileIndex = -1
1346 1346 self.pathList = []
1347 1347 self.filenameList = []
1348 1348 return
1349 1349
1350 1350 self.fileIndex = -1
1351 1351 self.pathList = pathList
1352 1352 self.filenameList = filenameList
1353 1353 file_name = os.path.basename(filenameList[-1])
1354 1354 basename, ext = os.path.splitext(file_name)
1355 1355 last_set = int(basename[-3:])
1356 1356
1357 1357 self.online = online
1358 1358 self.realtime = realtime
1359 1359 self.delay = delay
1360 1360 ext = ext.lower()
1361 1361 self.ext = ext
1362 1362 self.getByBlock = getblock
1363 1363 self.nTxs = nTxs
1364 1364 self.startTime = startTime
1365 1365 self.endTime = endTime
1366 1366 self.endDate = endDate
1367 1367 self.startDate = startDate
1368 1368 # Added-----------------
1369 1369 self.selBlocksize = blocksize
1370 1370 self.selBlocktime = blocktime
1371 1371
1372 1372 # Verbose-----------
1373 1373 self.verbose = verbose
1374 1374 self.warnings = warnings
1375 1375
1376 1376 if not(self.setNextFile()):
1377 1377 if (startDate != None) and (endDate != None):
1378 1378 print("[Reading] No files in range: %s - %s" % (datetime.datetime.combine(startDate, startTime).ctime(), datetime.datetime.combine(endDate, endTime).ctime()))
1379 1379 elif startDate != None:
1380 1380 print("[Reading] No files in range: %s" % (datetime.datetime.combine(startDate, startTime).ctime()))
1381 1381 else:
1382 1382 print("[Reading] No files")
1383 1383
1384 1384 self.fileIndex = -1
1385 1385 self.pathList = []
1386 1386 self.filenameList = []
1387 1387 return
1388 1388
1389 1389 # self.getBasicHeader()
1390 1390
1391 1391 if last_set != None:
1392 1392 self.dataOut.last_block = last_set * \
1393 1393 self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1394 1394 return
1395 1395
1396 1396 def getBasicHeader(self):
1397 1397
1398 1398 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond / \
1399 1399 1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1400 1400
1401 1401 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1402 1402
1403 1403 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1404 1404
1405 1405 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1406 1406
1407 1407 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1408 1408
1409 1409 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1410 1410
1411 1411 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
1412 1412
1413 1413 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1414 1414
1415 1415 def getFirstHeader(self):
1416 1416
1417 1417 raise NotImplementedError
1418 1418
1419 1419 def getData(self):
1420 1420
1421 1421 raise NotImplementedError
1422 1422
1423 1423 def hasNotDataInBuffer(self):
1424 1424
1425 1425 raise NotImplementedError
1426 1426
1427 1427 def readBlock(self):
1428 1428
1429 1429 raise NotImplementedError
1430 1430
1431 1431 def isEndProcess(self):
1432 1432
1433 1433 return self.flagNoMoreFiles
1434 1434
1435 1435 def printReadBlocks(self):
1436 1436
1437 1437 print("[Reading] Number of read blocks per file %04d" % self.nReadBlocks)
1438 1438
1439 1439 def printTotalBlocks(self):
1440 1440
1441 1441 print("[Reading] Number of read blocks %04d" % self.nTotalBlocks)
1442 1442
1443 1443 def printNumberOfBlock(self):
1444 1444 'SPAM!'
1445 1445
1446 1446 # if self.flagIsNewBlock:
1447 1447 # print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1448 1448 # self.processingHeaderObj.dataBlocksPerFile,
1449 1449 # self.dataOut.datatime.ctime())
1450 1450
1451 1451 def printInfo(self):
1452 1452
1453 1453 if self.__printInfo == False:
1454 1454 return
1455 1455
1456 1456 self.basicHeaderObj.printInfo()
1457 1457 self.systemHeaderObj.printInfo()
1458 1458 self.radarControllerHeaderObj.printInfo()
1459 1459 self.processingHeaderObj.printInfo()
1460 1460
1461 1461 self.__printInfo = False
1462 1462
1463 1463 def run(self,
1464 1464 path=None,
1465 1465 startDate=None,
1466 1466 endDate=None,
1467 1467 startTime=datetime.time(0, 0, 0),
1468 1468 endTime=datetime.time(23, 59, 59),
1469 1469 set=None,
1470 1470 expLabel="",
1471 1471 ext=None,
1472 1472 online=False,
1473 1473 delay=60,
1474 1474 walk=True,
1475 1475 getblock=False,
1476 1476 nTxs=1,
1477 1477 realtime=False,
1478 1478 blocksize=None,
1479 1479 blocktime=None,
1480 1480 skip=None,
1481 1481 cursor=None,
1482 1482 warnings=True,
1483 1483 server=None,
1484 1484 verbose=True,
1485 1485 format=None,
1486 1486 oneDDict=None,
1487 1487 twoDDict=None,
1488 1488 ind2DList=None, **kwargs):
1489 1489
1490 1490 if not(self.isConfig):
1491 1491 self.setup(path=path,
1492 1492 startDate=startDate,
1493 1493 endDate=endDate,
1494 1494 startTime=startTime,
1495 1495 endTime=endTime,
1496 1496 set=set,
1497 1497 expLabel=expLabel,
1498 1498 ext=ext,
1499 1499 online=online,
1500 1500 delay=delay,
1501 1501 walk=walk,
1502 1502 getblock=getblock,
1503 1503 nTxs=nTxs,
1504 1504 realtime=realtime,
1505 1505 blocksize=blocksize,
1506 1506 blocktime=blocktime,
1507 1507 skip=skip,
1508 1508 cursor=cursor,
1509 1509 warnings=warnings,
1510 1510 server=server,
1511 1511 verbose=verbose,
1512 1512 format=format,
1513 1513 oneDDict=oneDDict,
1514 1514 twoDDict=twoDDict,
1515 1515 ind2DList=ind2DList)
1516 1516 self.isConfig = True
1517 1517 if server is None:
1518 1518 self.getData()
1519 1519 else:
1520 1520 self.getFromServer()
1521 1521
1522 1522
1523 1523 class JRODataWriter(JRODataIO):
1524 1524
1525 1525 """
1526 1526 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1527 1527 de los datos siempre se realiza por bloques.
1528 1528 """
1529 1529
1530 1530 blockIndex = 0
1531 1531
1532 1532 path = None
1533 1533
1534 1534 setFile = None
1535 1535
1536 1536 profilesPerBlock = None
1537 1537
1538 1538 blocksPerFile = None
1539 1539
1540 1540 nWriteBlocks = 0
1541 1541
1542 1542 fileDate = None
1543 1543
1544 1544 def __init__(self, dataOut=None):
1545 1545 raise NotImplementedError
1546 1546
1547 1547 def hasAllDataInBuffer(self):
1548 1548 raise NotImplementedError
1549 1549
1550 1550 def setBlockDimension(self):
1551 1551 raise NotImplementedError
1552 1552
1553 1553 def writeBlock(self):
1554 1554 raise NotImplementedError
1555 1555
1556 1556 def putData(self):
1557 1557 raise NotImplementedError
1558 1558
1559 1559 def getProcessFlags(self):
1560 1560
1561 1561 processFlags = 0
1562 1562
1563 1563 dtype_index = get_dtype_index(self.dtype)
1564 1564 procflag_dtype = get_procflag_dtype(dtype_index)
1565 1565
1566 1566 processFlags += procflag_dtype
1567 1567
1568 1568 if self.dataOut.flagDecodeData:
1569 1569 processFlags += PROCFLAG.DECODE_DATA
1570 1570
1571 1571 if self.dataOut.flagDeflipData:
1572 1572 processFlags += PROCFLAG.DEFLIP_DATA
1573 1573
1574 1574 if self.dataOut.code is not None:
1575 1575 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1576 1576
1577 1577 if self.dataOut.nCohInt > 1:
1578 1578 processFlags += PROCFLAG.COHERENT_INTEGRATION
1579 1579
1580 1580 if self.dataOut.type == "Spectra":
1581 1581 if self.dataOut.nIncohInt > 1:
1582 1582 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1583 1583
1584 1584 if self.dataOut.data_dc is not None:
1585 1585 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1586 1586
1587 1587 if self.dataOut.flagShiftFFT:
1588 1588 processFlags += PROCFLAG.SHIFT_FFT_DATA
1589 1589
1590 1590 return processFlags
1591 1591
1592 1592 def setBasicHeader(self):
1593 1593
1594 1594 self.basicHeaderObj.size = self.basicHeaderSize # bytes
1595 1595 self.basicHeaderObj.version = self.versionFile
1596 1596 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1597 1597
1598 1598 utc = numpy.floor(self.dataOut.utctime)
1599 1599 milisecond = (self.dataOut.utctime - utc) * 1000.0
1600 1600
1601 1601 self.basicHeaderObj.utc = utc
1602 1602 self.basicHeaderObj.miliSecond = milisecond
1603 1603 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1604 1604 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1605 1605 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1606 1606
1607 1607 def setFirstHeader(self):
1608 1608 """
1609 1609 Obtiene una copia del First Header
1610 1610
1611 1611 Affected:
1612 1612
1613 1613 self.basicHeaderObj
1614 1614 self.systemHeaderObj
1615 1615 self.radarControllerHeaderObj
1616 1616 self.processingHeaderObj self.
1617 1617
1618 1618 Return:
1619 1619 None
1620 1620 """
1621 1621
1622 1622 raise NotImplementedError
1623 1623
1624 1624 def __writeFirstHeader(self):
1625 1625 """
1626 1626 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1627 1627
1628 1628 Affected:
1629 1629 __dataType
1630 1630
1631 1631 Return:
1632 1632 None
1633 1633 """
1634 1634
1635 1635 # CALCULAR PARAMETROS
1636 1636
1637 1637 sizeLongHeader = self.systemHeaderObj.size + \
1638 1638 self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1639 1639 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1640 1640
1641 1641 self.basicHeaderObj.write(self.fp)
1642 1642 self.systemHeaderObj.write(self.fp)
1643 1643 self.radarControllerHeaderObj.write(self.fp)
1644 1644 self.processingHeaderObj.write(self.fp)
1645 1645
1646 1646 def __setNewBlock(self):
1647 1647 """
1648 1648 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1649 1649
1650 1650 Return:
1651 1651 0 : si no pudo escribir nada
1652 1652 1 : Si escribio el Basic el First Header
1653 1653 """
1654 1654 if self.fp == None:
1655 1655 self.setNextFile()
1656 1656
1657 1657 if self.flagIsNewFile:
1658 1658 return 1
1659 1659
1660 1660 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1661 1661 self.basicHeaderObj.write(self.fp)
1662 1662 return 1
1663 1663
1664 1664 if not(self.setNextFile()):
1665 1665 return 0
1666 1666
1667 1667 return 1
1668 1668
1669 1669 def writeNextBlock(self):
1670 1670 """
1671 1671 Selecciona el bloque siguiente de datos y los escribe en un file
1672 1672
1673 1673 Return:
1674 1674 0 : Si no hizo pudo escribir el bloque de datos
1675 1675 1 : Si no pudo escribir el bloque de datos
1676 1676 """
1677 1677 if not(self.__setNewBlock()):
1678 1678 return 0
1679 1679
1680 1680 self.writeBlock()
1681 1681
1682 1682 print("[Writing] Block No. %d/%d" % (self.blockIndex,
1683 1683 self.processingHeaderObj.dataBlocksPerFile))
1684 1684
1685 1685 return 1
1686 1686
1687 1687 def setNextFile(self):
1688 1688 """
1689 1689 Determina el siguiente file que sera escrito
1690 1690
1691 1691 Affected:
1692 1692 self.filename
1693 1693 self.subfolder
1694 1694 self.fp
1695 1695 self.setFile
1696 1696 self.flagIsNewFile
1697 1697
1698 1698 Return:
1699 1699 0 : Si el archivo no puede ser escrito
1700 1700 1 : Si el archivo esta listo para ser escrito
1701 1701 """
1702 1702 ext = self.ext
1703 1703 path = self.path
1704 1704
1705 1705 if self.fp != None:
1706 1706 self.fp.close()
1707 1707
1708 1708 timeTuple = time.localtime(self.dataOut.utctime)
1709 1709 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year, timeTuple.tm_yday)
1710 1710
1711 1711 fullpath = os.path.join(path, subfolder)
1712 1712 setFile = self.setFile
1713 1713
1714 1714 if not(os.path.exists(fullpath)):
1715 1715 os.mkdir(fullpath)
1716 1716 setFile = -1 # inicializo mi contador de seteo
1717 1717 else:
1718 1718 filesList = os.listdir(fullpath)
1719 1719 if len(filesList) > 0:
1720 1720 filesList = sorted(filesList, key=str.lower)
1721 1721 filen = filesList[-1]
1722 1722 # el filename debera tener el siguiente formato
1723 1723 # 0 1234 567 89A BCDE (hex)
1724 1724 # x YYYY DDD SSS .ext
1725 1725 if isNumber(filen[8:11]):
1726 1726 # inicializo mi contador de seteo al seteo del ultimo file
1727 1727 setFile = int(filen[8:11])
1728 1728 else:
1729 1729 setFile = -1
1730 1730 else:
1731 1731 setFile = -1 # inicializo mi contador de seteo
1732 1732
1733 1733 setFile += 1
1734 1734
1735 1735 # If this is a new day it resets some values
1736 1736 if self.dataOut.datatime.date() > self.fileDate:
1737 1737 setFile = 0
1738 1738 self.nTotalBlocks = 0
1739 1739
1740 1740 filen = '{}{:04d}{:03d}{:03d}{}'.format(
1741 1741 self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext)
1742 1742
1743 1743 filename = os.path.join(path, subfolder, filen)
1744 1744
1745 1745 fp = open(filename, 'wb')
1746 1746
1747 1747 self.blockIndex = 0
1748 1748
1749 1749 # guardando atributos
1750 1750 self.filename = filename
1751 1751 self.subfolder = subfolder
1752 1752 self.fp = fp
1753 1753 self.setFile = setFile
1754 1754 self.flagIsNewFile = 1
1755 1755 self.fileDate = self.dataOut.datatime.date()
1756 1756
1757 1757 self.setFirstHeader()
1758 1758
1759 1759 print('[Writing] Opening file: %s' % self.filename)
1760 1760
1761 1761 self.__writeFirstHeader()
1762 1762
1763 1763 return 1
1764 1764
1765 1765 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1766 1766 """
1767 1767 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1768 1768
1769 1769 Inputs:
1770 1770 path : directory where data will be saved
1771 1771 profilesPerBlock : number of profiles per block
1772 1772 set : initial file set
1773 1773 datatype : An integer number that defines data type:
1774 1774 0 : int8 (1 byte)
1775 1775 1 : int16 (2 bytes)
1776 1776 2 : int32 (4 bytes)
1777 1777 3 : int64 (8 bytes)
1778 1778 4 : float32 (4 bytes)
1779 1779 5 : double64 (8 bytes)
1780 1780
1781 1781 Return:
1782 1782 0 : Si no realizo un buen seteo
1783 1783 1 : Si realizo un buen seteo
1784 1784 """
1785 1785
1786 1786 if ext == None:
1787 1787 ext = self.ext
1788 1788
1789 1789 self.ext = ext.lower()
1790 1790
1791 1791 self.path = path
1792 1792
1793 1793 if set is None:
1794 1794 self.setFile = -1
1795 1795 else:
1796 1796 self.setFile = set - 1
1797 1797
1798 1798 self.blocksPerFile = blocksPerFile
1799 1799
1800 1800 self.profilesPerBlock = profilesPerBlock
1801 1801
1802 1802 self.dataOut = dataOut
1803 1803 self.fileDate = self.dataOut.datatime.date()
1804 1804 # By default
1805 1805 self.dtype = self.dataOut.dtype
1806 1806
1807 1807 if datatype is not None:
1808 1808 self.dtype = get_numpy_dtype(datatype)
1809 1809
1810 1810 if not(self.setNextFile()):
1811 1811 print("[Writing] There isn't a next file")
1812 1812 return 0
1813 1813
1814 1814 self.setBlockDimension()
1815 1815
1816 1816 return 1
1817 1817
1818 1818 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1819 1819
1820 1820 if not(self.isConfig):
1821 1821
1822 1822 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock,
1823 1823 set=set, ext=ext, datatype=datatype, **kwargs)
1824 1824 self.isConfig = True
1825 1825
1826 1826 self.putData() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,679 +1,680
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import numpy
7 7
8 8 from schainpy.model.io.jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
9 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
10 10 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
11 11 from schainpy.model.data.jrodata import Spectra
12 12
13 @MPDecorator
13 14 class SpectraReader(JRODataReader, ProcessingUnit):
14 15 """
15 16 Esta clase permite leer datos de espectros desde archivos procesados (.pdata). La lectura
16 17 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones)
17 18 son almacenados en tres buffer's para el Self Spectra, el Cross Spectra y el DC Channel.
18 19
19 20 paresCanalesIguales * alturas * perfiles (Self Spectra)
20 21 paresCanalesDiferentes * alturas * perfiles (Cross Spectra)
21 22 canales * alturas (DC Channels)
22 23
23 24 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
24 25 RadarControllerHeader y Spectra. Los tres primeros se usan para almacenar informacion de la
25 26 cabecera de datos (metadata), y el cuarto (Spectra) para obtener y almacenar un bloque de
26 27 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
27 28
28 29 Example:
29 30 dpath = "/home/myuser/data"
30 31
31 32 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
32 33
33 34 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
34 35
35 36 readerObj = SpectraReader()
36 37
37 38 readerObj.setup(dpath, startTime, endTime)
38 39
39 40 while(True):
40 41
41 42 readerObj.getData()
42 43
43 44 print readerObj.data_spc
44 45
45 46 print readerObj.data_cspc
46 47
47 48 print readerObj.data_dc
48 49
49 50 if readerObj.flagNoMoreFiles:
50 51 break
51 52
52 53 """
53 54
54 55 pts2read_SelfSpectra = 0
55 56
56 57 pts2read_CrossSpectra = 0
57 58
58 59 pts2read_DCchannels = 0
59 60
60 61 ext = ".pdata"
61 62
62 63 optchar = "P"
63 64
64 65 dataOut = None
65 66
66 67 nRdChannels = None
67 68
68 69 nRdPairs = None
69 70
70 71 rdPairList = []
71 72
72 def __init__(self, **kwargs):
73 def __init__(self):#, **kwargs):
73 74 """
74 75 Inicializador de la clase SpectraReader para la lectura de datos de espectros.
75 76
76 77 Inputs:
77 78 dataOut : Objeto de la clase Spectra. Este objeto sera utilizado para
78 79 almacenar un perfil de datos cada vez que se haga un requerimiento
79 80 (getData). El perfil sera obtenido a partir del buffer de datos,
80 81 si el buffer esta vacio se hara un nuevo proceso de lectura de un
81 82 bloque de datos.
82 83 Si este parametro no es pasado se creara uno internamente.
83 84
84 85 Affected:
85 86 self.dataOut
86 87
87 88 Return : None
88 89 """
89 90
90 91 #Eliminar de la base la herencia
91 ProcessingUnit.__init__(self, **kwargs)
92 ProcessingUnit.__init__(self)#, **kwargs)
92 93
93 94 # self.isConfig = False
94 95
95 96 self.pts2read_SelfSpectra = 0
96 97
97 98 self.pts2read_CrossSpectra = 0
98 99
99 100 self.pts2read_DCchannels = 0
100 101
101 102 self.datablock = None
102 103
103 104 self.utc = None
104 105
105 106 self.ext = ".pdata"
106 107
107 108 self.optchar = "P"
108 109
109 110 self.basicHeaderObj = BasicHeader(LOCALTIME)
110 111
111 112 self.systemHeaderObj = SystemHeader()
112 113
113 114 self.radarControllerHeaderObj = RadarControllerHeader()
114 115
115 116 self.processingHeaderObj = ProcessingHeader()
116 117
117 118 self.online = 0
118 119
119 120 self.fp = None
120 121
121 122 self.idFile = None
122 123
123 124 self.dtype = None
124 125
125 126 self.fileSizeByHeader = None
126 127
127 128 self.filenameList = []
128 129
129 130 self.filename = None
130 131
131 132 self.fileSize = None
132 133
133 134 self.firstHeaderSize = 0
134 135
135 136 self.basicHeaderSize = 24
136 137
137 138 self.pathList = []
138 139
139 140 self.lastUTTime = 0
140 141
141 142 self.maxTimeStep = 30
142 143
143 144 self.flagNoMoreFiles = 0
144 145
145 146 self.set = 0
146 147
147 148 self.path = None
148 149
149 150 self.delay = 60 #seconds
150 151
151 152 self.nTries = 3 #quantity tries
152 153
153 154 self.nFiles = 3 #number of files for searching
154 155
155 156 self.nReadBlocks = 0
156 157
157 158 self.flagIsNewFile = 1
158 159
159 160 self.__isFirstTimeOnline = 1
160 161
161 162 # self.ippSeconds = 0
162 163
163 164 self.flagDiscontinuousBlock = 0
164 165
165 166 self.flagIsNewBlock = 0
166 167
167 168 self.nTotalBlocks = 0
168 169
169 170 self.blocksize = 0
170 171
171 172 self.dataOut = self.createObjByDefault()
172 173
173 174 self.profileIndex = 1 #Always
174 175
175 176
176 177 def createObjByDefault(self):
177 178
178 179 dataObj = Spectra()
179 180
180 181 return dataObj
181 182
182 183 def __hasNotDataInBuffer(self):
183 184 return 1
184 185
185 186
186 187 def getBlockDimension(self):
187 188 """
188 189 Obtiene la cantidad de puntos a leer por cada bloque de datos
189 190
190 191 Affected:
191 192 self.nRdChannels
192 193 self.nRdPairs
193 194 self.pts2read_SelfSpectra
194 195 self.pts2read_CrossSpectra
195 196 self.pts2read_DCchannels
196 197 self.blocksize
197 198 self.dataOut.nChannels
198 199 self.dataOut.nPairs
199 200
200 201 Return:
201 202 None
202 203 """
203 204 self.nRdChannels = 0
204 205 self.nRdPairs = 0
205 206 self.rdPairList = []
206 207
207 208 for i in range(0, self.processingHeaderObj.totalSpectra*2, 2):
208 209 if self.processingHeaderObj.spectraComb[i] == self.processingHeaderObj.spectraComb[i+1]:
209 210 self.nRdChannels = self.nRdChannels + 1 #par de canales iguales
210 211 else:
211 212 self.nRdPairs = self.nRdPairs + 1 #par de canales diferentes
212 213 self.rdPairList.append((self.processingHeaderObj.spectraComb[i], self.processingHeaderObj.spectraComb[i+1]))
213 214
214 215 pts2read = self.processingHeaderObj.nHeights * self.processingHeaderObj.profilesPerBlock
215 216
216 217 self.pts2read_SelfSpectra = int(self.nRdChannels * pts2read)
217 218 self.blocksize = self.pts2read_SelfSpectra
218 219
219 220 if self.processingHeaderObj.flag_cspc:
220 221 self.pts2read_CrossSpectra = int(self.nRdPairs * pts2read)
221 222 self.blocksize += self.pts2read_CrossSpectra
222 223
223 224 if self.processingHeaderObj.flag_dc:
224 225 self.pts2read_DCchannels = int(self.systemHeaderObj.nChannels * self.processingHeaderObj.nHeights)
225 226 self.blocksize += self.pts2read_DCchannels
226 227
227 228 # self.blocksize = self.pts2read_SelfSpectra + self.pts2read_CrossSpectra + self.pts2read_DCchannels
228 229
229 230
230 231 def readBlock(self):
231 232 """
232 233 Lee el bloque de datos desde la posicion actual del puntero del archivo
233 234 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
234 235 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
235 236 es seteado a 0
236 237
237 238 Return: None
238 239
239 240 Variables afectadas:
240 241
241 242 self.flagIsNewFile
242 243 self.flagIsNewBlock
243 244 self.nTotalBlocks
244 245 self.data_spc
245 246 self.data_cspc
246 247 self.data_dc
247 248
248 249 Exceptions:
249 250 Si un bloque leido no es un bloque valido
250 251 """
251 252 blockOk_flag = False
252 253 fpointer = self.fp.tell()
253 254
254 255 spc = numpy.fromfile( self.fp, self.dtype[0], self.pts2read_SelfSpectra )
255 256 spc = spc.reshape( (self.nRdChannels, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
256 257
257 258 if self.processingHeaderObj.flag_cspc:
258 259 cspc = numpy.fromfile( self.fp, self.dtype, self.pts2read_CrossSpectra )
259 260 cspc = cspc.reshape( (self.nRdPairs, self.processingHeaderObj.nHeights, self.processingHeaderObj.profilesPerBlock) ) #transforma a un arreglo 3D
260 261
261 262 if self.processingHeaderObj.flag_dc:
262 263 dc = numpy.fromfile( self.fp, self.dtype, self.pts2read_DCchannels ) #int(self.processingHeaderObj.nHeights*self.systemHeaderObj.nChannels) )
263 264 dc = dc.reshape( (self.systemHeaderObj.nChannels, self.processingHeaderObj.nHeights) ) #transforma a un arreglo 2D
264 265
265 266
266 267 if not self.processingHeaderObj.shif_fft:
267 268 #desplaza a la derecha en el eje 2 determinadas posiciones
268 269 shift = int(self.processingHeaderObj.profilesPerBlock/2)
269 270 spc = numpy.roll( spc, shift , axis=2 )
270 271
271 272 if self.processingHeaderObj.flag_cspc:
272 273 #desplaza a la derecha en el eje 2 determinadas posiciones
273 274 cspc = numpy.roll( cspc, shift, axis=2 )
274 275
275 276 #Dimensions : nChannels, nProfiles, nSamples
276 277 spc = numpy.transpose( spc, (0,2,1) )
277 278 self.data_spc = spc
278 279
279 280 if self.processingHeaderObj.flag_cspc:
280 281 cspc = numpy.transpose( cspc, (0,2,1) )
281 282 self.data_cspc = cspc['real'] + cspc['imag']*1j
282 283 else:
283 284 self.data_cspc = None
284 285
285 286 if self.processingHeaderObj.flag_dc:
286 287 self.data_dc = dc['real'] + dc['imag']*1j
287 288 else:
288 289 self.data_dc = None
289 290
290 291 self.flagIsNewFile = 0
291 292 self.flagIsNewBlock = 1
292 293
293 294 self.nTotalBlocks += 1
294 295 self.nReadBlocks += 1
295 296
296 297 return 1
297 298
298 299 def getFirstHeader(self):
299 300
300 301 self.getBasicHeader()
301 302
302 303 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
303 304
304 305 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
305 306
306 307 # self.dataOut.ippSeconds = self.ippSeconds
307 308
308 309 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt * self.processingHeaderObj.nIncohInt * self.processingHeaderObj.profilesPerBlock
309 310
310 311 self.dataOut.dtype = self.dtype
311 312
312 313 # self.dataOut.nPairs = self.nPairs
313 314
314 315 self.dataOut.pairsList = self.rdPairList
315 316
316 317 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
317 318
318 319 self.dataOut.nFFTPoints = self.processingHeaderObj.profilesPerBlock
319 320
320 321 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
321 322
322 323 self.dataOut.nIncohInt = self.processingHeaderObj.nIncohInt
323 324
324 325 xf = self.processingHeaderObj.firstHeight + self.processingHeaderObj.nHeights*self.processingHeaderObj.deltaHeight
325 326
326 327 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.firstHeight, xf, self.processingHeaderObj.deltaHeight)
327 328
328 329 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
329 330
330 331 self.dataOut.flagShiftFFT = True #Data is always shifted
331 332
332 333 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
333 334
334 335 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data esta sin flip
335 336
336 337 def getData(self):
337 338 """
338 339 First method to execute before "RUN" is called.
339 340
340 341 Copia el buffer de lectura a la clase "Spectra",
341 342 con todos los parametros asociados a este (metadata). cuando no hay datos en el buffer de
342 343 lectura es necesario hacer una nueva lectura de los bloques de datos usando "readNextBlock"
343 344
344 345 Return:
345 346 0 : Si no hay mas archivos disponibles
346 347 1 : Si hizo una buena copia del buffer
347 348
348 349 Affected:
349 350 self.dataOut
350 351
351 352 self.flagDiscontinuousBlock
352 353 self.flagIsNewBlock
353 354 """
354 355
355 356 if self.flagNoMoreFiles:
356 357 self.dataOut.flagNoData = True
357 358 print('Process finished')
358 359 return 0
359 360
360 361 self.flagDiscontinuousBlock = 0
361 362 self.flagIsNewBlock = 0
362 363
363 364 if self.__hasNotDataInBuffer():
364 365
365 366 if not( self.readNextBlock() ):
366 367 self.dataOut.flagNoData = True
367 368 return 0
368 369
369 370 #data es un numpy array de 3 dmensiones (perfiles, alturas y canales)
370 371
371 372 if self.data_spc is None:
372 373 self.dataOut.flagNoData = True
373 374 return 0
374 375
375 376 self.getBasicHeader()
376 377
377 378 self.getFirstHeader()
378 379
379 380 self.dataOut.data_spc = self.data_spc
380 381
381 382 self.dataOut.data_cspc = self.data_cspc
382 383
383 384 self.dataOut.data_dc = self.data_dc
384 385
385 386 self.dataOut.flagNoData = False
386 387
387 388 self.dataOut.realtime = self.online
388 389
389 390 return self.dataOut.data_spc
390 391
391 392 class SpectraWriter(JRODataWriter, Operation):
392 393
393 394 """
394 395 Esta clase permite escribir datos de espectros a archivos procesados (.pdata). La escritura
395 396 de los datos siempre se realiza por bloques.
396 397 """
397 398
398 399 ext = ".pdata"
399 400
400 401 optchar = "P"
401 402
402 403 shape_spc_Buffer = None
403 404
404 405 shape_cspc_Buffer = None
405 406
406 407 shape_dc_Buffer = None
407 408
408 409 data_spc = None
409 410
410 411 data_cspc = None
411 412
412 413 data_dc = None
413 414
414 415 # dataOut = None
415 416
416 417 def __init__(self, **kwargs):
417 418 """
418 419 Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
419 420
420 421 Affected:
421 422 self.dataOut
422 423 self.basicHeaderObj
423 424 self.systemHeaderObj
424 425 self.radarControllerHeaderObj
425 426 self.processingHeaderObj
426 427
427 428 Return: None
428 429 """
429 430
430 431 Operation.__init__(self, **kwargs)
431 432
432 433 self.isConfig = False
433 434
434 435 self.nTotalBlocks = 0
435 436
436 437 self.data_spc = None
437 438
438 439 self.data_cspc = None
439 440
440 441 self.data_dc = None
441 442
442 443 self.fp = None
443 444
444 445 self.flagIsNewFile = 1
445 446
446 447 self.nTotalBlocks = 0
447 448
448 449 self.flagIsNewBlock = 0
449 450
450 451 self.setFile = None
451 452
452 453 self.dtype = None
453 454
454 455 self.path = None
455 456
456 457 self.noMoreFiles = 0
457 458
458 459 self.filename = None
459 460
460 461 self.basicHeaderObj = BasicHeader(LOCALTIME)
461 462
462 463 self.systemHeaderObj = SystemHeader()
463 464
464 465 self.radarControllerHeaderObj = RadarControllerHeader()
465 466
466 467 self.processingHeaderObj = ProcessingHeader()
467 468
468 469
469 470 def hasAllDataInBuffer(self):
470 471 return 1
471 472
472 473
473 474 def setBlockDimension(self):
474 475 """
475 476 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
476 477
477 478 Affected:
478 479 self.shape_spc_Buffer
479 480 self.shape_cspc_Buffer
480 481 self.shape_dc_Buffer
481 482
482 483 Return: None
483 484 """
484 485 self.shape_spc_Buffer = (self.dataOut.nChannels,
485 486 self.processingHeaderObj.nHeights,
486 487 self.processingHeaderObj.profilesPerBlock)
487 488
488 489 self.shape_cspc_Buffer = (self.dataOut.nPairs,
489 490 self.processingHeaderObj.nHeights,
490 491 self.processingHeaderObj.profilesPerBlock)
491 492
492 493 self.shape_dc_Buffer = (self.dataOut.nChannels,
493 494 self.processingHeaderObj.nHeights)
494 495
495 496
496 497 def writeBlock(self):
497 498 """
498 499 Escribe el buffer en el file designado
499 500
500 501 Affected:
501 502 self.data_spc
502 503 self.data_cspc
503 504 self.data_dc
504 505 self.flagIsNewFile
505 506 self.flagIsNewBlock
506 507 self.nTotalBlocks
507 508 self.nWriteBlocks
508 509
509 510 Return: None
510 511 """
511 512
512 513 spc = numpy.transpose( self.data_spc, (0,2,1) )
513 514 if not self.processingHeaderObj.shif_fft:
514 515 spc = numpy.roll( spc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
515 516 data = spc.reshape((-1))
516 517 data = data.astype(self.dtype[0])
517 518 data.tofile(self.fp)
518 519
519 520 if self.data_cspc is not None:
520 521 data = numpy.zeros( self.shape_cspc_Buffer, self.dtype )
521 522 cspc = numpy.transpose( self.data_cspc, (0,2,1) )
522 523 if not self.processingHeaderObj.shif_fft:
523 524 cspc = numpy.roll( cspc, self.processingHeaderObj.profilesPerBlock/2, axis=2 ) #desplaza a la derecha en el eje 2 determinadas posiciones
524 525 data['real'] = cspc.real
525 526 data['imag'] = cspc.imag
526 527 data = data.reshape((-1))
527 528 data.tofile(self.fp)
528 529
529 530 if self.data_dc is not None:
530 531 data = numpy.zeros( self.shape_dc_Buffer, self.dtype )
531 532 dc = self.data_dc
532 533 data['real'] = dc.real
533 534 data['imag'] = dc.imag
534 535 data = data.reshape((-1))
535 536 data.tofile(self.fp)
536 537
537 538 # self.data_spc.fill(0)
538 539 #
539 540 # if self.data_dc is not None:
540 541 # self.data_dc.fill(0)
541 542 #
542 543 # if self.data_cspc is not None:
543 544 # self.data_cspc.fill(0)
544 545
545 546 self.flagIsNewFile = 0
546 547 self.flagIsNewBlock = 1
547 548 self.nTotalBlocks += 1
548 549 self.nWriteBlocks += 1
549 550 self.blockIndex += 1
550 551
551 552 # print "[Writing] Block = %d04" %self.blockIndex
552 553
553 554 def putData(self):
554 555 """
555 556 Setea un bloque de datos y luego los escribe en un file
556 557
557 558 Affected:
558 559 self.data_spc
559 560 self.data_cspc
560 561 self.data_dc
561 562
562 563 Return:
563 564 0 : Si no hay data o no hay mas files que puedan escribirse
564 565 1 : Si se escribio la data de un bloque en un file
565 566 """
566 567
567 568 if self.dataOut.flagNoData:
568 569 return 0
569 570
570 571 self.flagIsNewBlock = 0
571 572
572 573 if self.dataOut.flagDiscontinuousBlock:
573 574 self.data_spc.fill(0)
574 575 if self.dataOut.data_cspc is not None:
575 576 self.data_cspc.fill(0)
576 577 if self.dataOut.data_dc is not None:
577 578 self.data_dc.fill(0)
578 579 self.setNextFile()
579 580
580 581 if self.flagIsNewFile == 0:
581 582 self.setBasicHeader()
582 583
583 584 self.data_spc = self.dataOut.data_spc.copy()
584 585
585 586 if self.dataOut.data_cspc is not None:
586 587 self.data_cspc = self.dataOut.data_cspc.copy()
587 588
588 589 if self.dataOut.data_dc is not None:
589 590 self.data_dc = self.dataOut.data_dc.copy()
590 591
591 592 # #self.processingHeaderObj.dataBlocksPerFile)
592 593 if self.hasAllDataInBuffer():
593 594 # self.setFirstHeader()
594 595 self.writeNextBlock()
595 596
596 597 return 1
597 598
598 599 def __getBlockSize(self):
599 600 '''
600 601 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra
601 602 '''
602 603
603 604 dtype_width = self.getDtypeWidth()
604 605
605 606 pts2write = self.dataOut.nHeights * self.dataOut.nFFTPoints
606 607
607 608 pts2write_SelfSpectra = int(self.dataOut.nChannels * pts2write)
608 609 blocksize = (pts2write_SelfSpectra*dtype_width)
609 610
610 611 if self.dataOut.data_cspc is not None:
611 612 pts2write_CrossSpectra = int(self.dataOut.nPairs * pts2write)
612 613 blocksize += (pts2write_CrossSpectra*dtype_width*2)
613 614
614 615 if self.dataOut.data_dc is not None:
615 616 pts2write_DCchannels = int(self.dataOut.nChannels * self.dataOut.nHeights)
616 617 blocksize += (pts2write_DCchannels*dtype_width*2)
617 618
618 619 # blocksize = blocksize #* datatypeValue * 2 #CORREGIR ESTO
619 620
620 621 return blocksize
621 622
622 623 def setFirstHeader(self):
623 624
624 625 """
625 626 Obtiene una copia del First Header
626 627
627 628 Affected:
628 629 self.systemHeaderObj
629 630 self.radarControllerHeaderObj
630 631 self.dtype
631 632
632 633 Return:
633 634 None
634 635 """
635 636
636 637 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
637 638 self.systemHeaderObj.nChannels = self.dataOut.nChannels
638 639 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
639 640
640 641 self.processingHeaderObj.dtype = 1 # Spectra
641 642 self.processingHeaderObj.blockSize = self.__getBlockSize()
642 643 self.processingHeaderObj.profilesPerBlock = self.dataOut.nFFTPoints
643 644 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
644 645 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
645 646 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt# Se requiere para determinar el valor de timeInterval
646 647 self.processingHeaderObj.nIncohInt = self.dataOut.nIncohInt
647 648 self.processingHeaderObj.totalSpectra = self.dataOut.nPairs + self.dataOut.nChannels
648 649 self.processingHeaderObj.shif_fft = self.dataOut.flagShiftFFT
649 650
650 651 if self.processingHeaderObj.totalSpectra > 0:
651 652 channelList = []
652 653 for channel in range(self.dataOut.nChannels):
653 654 channelList.append(channel)
654 655 channelList.append(channel)
655 656
656 657 pairsList = []
657 658 if self.dataOut.nPairs > 0:
658 659 for pair in self.dataOut.pairsList:
659 660 pairsList.append(pair[0])
660 661 pairsList.append(pair[1])
661 662
662 663 spectraComb = channelList + pairsList
663 664 spectraComb = numpy.array(spectraComb, dtype="u1")
664 665 self.processingHeaderObj.spectraComb = spectraComb
665 666
666 667 if self.dataOut.code is not None:
667 668 self.processingHeaderObj.code = self.dataOut.code
668 669 self.processingHeaderObj.nCode = self.dataOut.nCode
669 670 self.processingHeaderObj.nBaud = self.dataOut.nBaud
670 671
671 672 if self.processingHeaderObj.nWindows != 0:
672 673 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
673 674 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
674 675 self.processingHeaderObj.nHeights = self.dataOut.nHeights
675 676 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
676 677
677 678 self.processingHeaderObj.processFlags = self.getProcessFlags()
678 679
679 680 self.setBasicHeader() No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,764 +1,765
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from .jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 13 import zmq
14 14 import tempfile
15 15 from io import StringIO
16 16 # from _sha import blocksize
17 17
18
18 @MPDecorator
19 19 class VoltageReader(JRODataReader, ProcessingUnit):
20 20 """
21 21 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
22 22 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
23 23 perfiles*alturas*canales) son almacenados en la variable "buffer".
24 24
25 25 perfiles * alturas * canales
26 26
27 27 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
28 28 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
29 29 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
30 30 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
31 31
32 32 Example:
33 33
34 34 dpath = "/home/myuser/data"
35 35
36 36 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
37 37
38 38 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
39 39
40 40 readerObj = VoltageReader()
41 41
42 42 readerObj.setup(dpath, startTime, endTime)
43 43
44 44 while(True):
45 45
46 46 #to get one profile
47 47 profile = readerObj.getData()
48 48
49 49 #print the profile
50 50 print profile
51 51
52 52 #If you want to see all datablock
53 53 print readerObj.datablock
54 54
55 55 if readerObj.flagNoMoreFiles:
56 56 break
57 57
58 58 """
59 59
60 60 ext = ".r"
61 61
62 62 optchar = "D"
63 63 dataOut = None
64 64
65 def __init__(self, **kwargs):
65 def __init__(self):#, **kwargs):
66 66 """
67 67 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
68 68
69 69 Input:
70 70 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
71 71 almacenar un perfil de datos cada vez que se haga un requerimiento
72 72 (getData). El perfil sera obtenido a partir del buffer de datos,
73 73 si el buffer esta vacio se hara un nuevo proceso de lectura de un
74 74 bloque de datos.
75 75 Si este parametro no es pasado se creara uno internamente.
76 76
77 77 Variables afectadas:
78 78 self.dataOut
79 79
80 80 Return:
81 81 None
82 82 """
83 83
84 ProcessingUnit.__init__(self, **kwargs)
84 ProcessingUnit.__init__(self)#, **kwargs)
85 85
86 86 self.isConfig = False
87 87
88 88 self.datablock = None
89 89
90 90 self.utc = 0
91 91
92 92 self.ext = ".r"
93 93
94 94 self.optchar = "D"
95 95
96 96 self.basicHeaderObj = BasicHeader(LOCALTIME)
97 97
98 98 self.systemHeaderObj = SystemHeader()
99 99
100 100 self.radarControllerHeaderObj = RadarControllerHeader()
101 101
102 102 self.processingHeaderObj = ProcessingHeader()
103 103
104 104 self.online = 0
105 105
106 106 self.fp = None
107 107
108 108 self.idFile = None
109 109
110 110 self.dtype = None
111 111
112 112 self.fileSizeByHeader = None
113 113
114 114 self.filenameList = []
115 115
116 116 self.filename = None
117 117
118 118 self.fileSize = None
119 119
120 120 self.firstHeaderSize = 0
121 121
122 122 self.basicHeaderSize = 24
123 123
124 124 self.pathList = []
125 125
126 126 self.filenameList = []
127 127
128 128 self.lastUTTime = 0
129 129
130 130 self.maxTimeStep = 30
131 131
132 132 self.flagNoMoreFiles = 0
133 133
134 134 self.set = 0
135 135
136 136 self.path = None
137 137
138 138 self.profileIndex = 2**32 - 1
139 139
140 140 self.delay = 3 # seconds
141 141
142 142 self.nTries = 3 # quantity tries
143 143
144 144 self.nFiles = 3 # number of files for searching
145 145
146 146 self.nReadBlocks = 0
147 147
148 148 self.flagIsNewFile = 1
149 149
150 150 self.__isFirstTimeOnline = 1
151 151
152 152 # self.ippSeconds = 0
153 153
154 154 self.flagDiscontinuousBlock = 0
155 155
156 156 self.flagIsNewBlock = 0
157 157
158 158 self.nTotalBlocks = 0
159 159
160 160 self.blocksize = 0
161 161
162 162 self.dataOut = self.createObjByDefault()
163 163
164 164 self.nTxs = 1
165 165
166 166 self.txIndex = 0
167 167
168 168 def createObjByDefault(self):
169 169
170 170 dataObj = Voltage()
171 171
172 172 return dataObj
173 173
174 174 def __hasNotDataInBuffer(self):
175 175
176 176 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
177 177 return 1
178 178
179 179 return 0
180 180
181 181 def getBlockDimension(self):
182 182 """
183 183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184 184
185 185 Affected:
186 186 self.blocksize
187 187
188 188 Return:
189 189 None
190 190 """
191 191 pts2read = self.processingHeaderObj.profilesPerBlock * \
192 192 self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
193 193 self.blocksize = pts2read
194 194
195 195 def readBlock(self):
196 196 """
197 197 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
198 198 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
199 199 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
200 200 es seteado a 0
201 201
202 202 Inputs:
203 203 None
204 204
205 205 Return:
206 206 None
207 207
208 208 Affected:
209 209 self.profileIndex
210 210 self.datablock
211 211 self.flagIsNewFile
212 212 self.flagIsNewBlock
213 213 self.nTotalBlocks
214 214
215 215 Exceptions:
216 216 Si un bloque leido no es un bloque valido
217 217 """
218 218
219 219 # if self.server is not None:
220 220 # self.zBlock = self.receiver.recv()
221 221 # self.zHeader = self.zBlock[:24]
222 222 # self.zDataBlock = self.zBlock[24:]
223 223 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
224 224 # self.processingHeaderObj.profilesPerBlock = 240
225 225 # self.processingHeaderObj.nHeights = 248
226 226 # self.systemHeaderObj.nChannels
227 227 # else:
228 228 current_pointer_location = self.fp.tell()
229 229 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
230 230
231 231 try:
232 232 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
233 233 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
234 234 except:
235 235 # print "The read block (%3d) has not enough data" %self.nReadBlocks
236 236
237 237 if self.waitDataBlock(pointer_location=current_pointer_location):
238 238 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
239 239 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
240 240 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
241 241 # return 0
242 242
243 243 # Dimensions : nChannels, nProfiles, nSamples
244 244
245 245 junk = numpy.transpose(junk, (2, 0, 1))
246 246 self.datablock = junk['real'] + junk['imag'] * 1j
247 247
248 248 self.profileIndex = 0
249 249
250 250 self.flagIsNewFile = 0
251 251 self.flagIsNewBlock = 1
252 252
253 253 self.nTotalBlocks += 1
254 254 self.nReadBlocks += 1
255 255
256 256 return 1
257 257
258 258 def getFirstHeader(self):
259 259
260 260 self.getBasicHeader()
261 261
262 262 self.dataOut.processingHeaderObj = self.processingHeaderObj.copy()
263 263
264 264 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
265 265
266 266 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
267 267
268 268 if self.nTxs > 1:
269 269 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
270 270 # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
271 271
272 272 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
273 273 #
274 274 # if self.radarControllerHeaderObj.code is not None:
275 275 #
276 276 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
277 277 #
278 278 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
279 279 #
280 280 # self.dataOut.code = self.radarControllerHeaderObj.code
281 281
282 282 self.dataOut.dtype = self.dtype
283 283
284 284 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
285 285
286 286 self.dataOut.heightList = numpy.arange(
287 287 self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
288 288
289 289 self.dataOut.channelList = list(range(self.systemHeaderObj.nChannels))
290 290
291 291 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
292 292
293 293 # asumo q la data no esta decodificada
294 294 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
295 295
296 296 # asumo q la data no esta sin flip
297 297 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
298 298
299 299 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
300 300
301 301 def reshapeData(self):
302 302
303 303 if self.nTxs < 0:
304 304 return
305 305
306 306 if self.nTxs == 1:
307 307 return
308 308
309 309 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
310 310 raise ValueError("1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
311 311 1. / self.nTxs, self.processingHeaderObj.profilesPerBlock))
312 312
313 313 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
314 314 raise ValueError("nTxs (=%d), should be a multiple of nHeights (=%d)" % (
315 315 self.nTxs, self.processingHeaderObj.nHeights))
316 316
317 317 self.datablock = self.datablock.reshape(
318 318 (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, self.processingHeaderObj.nHeights / self.nTxs))
319 319
320 320 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
321 321 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
322 322 self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
323 323 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
324 324
325 325 return
326 326
327 327 def readFirstHeaderFromServer(self):
328 328
329 329 self.getFirstHeader()
330 330
331 331 self.firstHeaderSize = self.basicHeaderObj.size
332 332
333 333 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
334 334 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
335 335 if datatype == 0:
336 336 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
337 337 elif datatype == 1:
338 338 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
339 339 elif datatype == 2:
340 340 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
341 341 elif datatype == 3:
342 342 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
343 343 elif datatype == 4:
344 344 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
345 345 elif datatype == 5:
346 346 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
347 347 else:
348 348 raise ValueError('Data type was not defined')
349 349
350 350 self.dtype = datatype_str
351 351 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
352 352 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
353 353 self.firstHeaderSize + self.basicHeaderSize * \
354 354 (self.processingHeaderObj.dataBlocksPerFile - 1)
355 355 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
356 356 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
357 357 self.getBlockDimension()
358 358
359 359 def getFromServer(self):
360 360 self.flagDiscontinuousBlock = 0
361 361 self.profileIndex = 0
362 362 self.flagIsNewBlock = 1
363 363 self.dataOut.flagNoData = False
364 364 self.nTotalBlocks += 1
365 365 self.nReadBlocks += 1
366 366 self.blockPointer = 0
367 367
368 368 block = self.receiver.recv()
369 369
370 370 self.basicHeaderObj.read(block[self.blockPointer:])
371 371 self.blockPointer += self.basicHeaderObj.length
372 372 self.systemHeaderObj.read(block[self.blockPointer:])
373 373 self.blockPointer += self.systemHeaderObj.length
374 374 self.radarControllerHeaderObj.read(block[self.blockPointer:])
375 375 self.blockPointer += self.radarControllerHeaderObj.length
376 376 self.processingHeaderObj.read(block[self.blockPointer:])
377 377 self.blockPointer += self.processingHeaderObj.length
378 378 self.readFirstHeaderFromServer()
379 379
380 380 timestamp = self.basicHeaderObj.get_datatime()
381 381 print('[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp))
382 382 current_pointer_location = self.blockPointer
383 383 junk = numpy.fromstring(
384 384 block[self.blockPointer:], self.dtype, self.blocksize)
385 385
386 386 try:
387 387 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
388 388 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
389 389 except:
390 390 # print "The read block (%3d) has not enough data" %self.nReadBlocks
391 391 if self.waitDataBlock(pointer_location=current_pointer_location):
392 392 junk = numpy.fromstring(
393 393 block[self.blockPointer:], self.dtype, self.blocksize)
394 394 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
395 395 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
396 396 # return 0
397 397
398 398 # Dimensions : nChannels, nProfiles, nSamples
399 399
400 400 junk = numpy.transpose(junk, (2, 0, 1))
401 401 self.datablock = junk['real'] + junk['imag'] * 1j
402 402 self.profileIndex = 0
403 403 if self.selBlocksize == None:
404 404 self.selBlocksize = self.dataOut.nProfiles
405 405 if self.selBlocktime != None:
406 406 if self.dataOut.nCohInt is not None:
407 407 nCohInt = self.dataOut.nCohInt
408 408 else:
409 409 nCohInt = 1
410 410 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
411 411 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
412 412 self.dataOut.data = self.datablock[:,
413 413 self.profileIndex:self.profileIndex + self.selBlocksize, :]
414 414 datasize = self.dataOut.data.shape[1]
415 415 if datasize < self.selBlocksize:
416 416 buffer = numpy.zeros(
417 417 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
418 418 buffer[:, :datasize, :] = self.dataOut.data
419 419 self.dataOut.data = buffer
420 420 self.profileIndex = blockIndex
421 421
422 422 self.dataOut.flagDataAsBlock = True
423 423 self.flagIsNewBlock = 1
424 424 self.dataOut.realtime = self.online
425 425
426 426 return self.dataOut.data
427 427
428 428 def getData(self):
429 429 """
430 430 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
431 431 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
432 432 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
433 433 "readNextBlock"
434 434
435 435 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
436 436
437 437 Return:
438 438
439 439 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
440 440 es igual al total de perfiles leidos desde el archivo.
441 441
442 442 Si self.getByBlock == False:
443 443
444 444 self.dataOut.data = buffer[:, thisProfile, :]
445 445
446 446 shape = [nChannels, nHeis]
447 447
448 448 Si self.getByBlock == True:
449 449
450 450 self.dataOut.data = buffer[:, :, :]
451 451
452 452 shape = [nChannels, nProfiles, nHeis]
453 453
454 454 Variables afectadas:
455 455 self.dataOut
456 456 self.profileIndex
457 457
458 458 Affected:
459 459 self.dataOut
460 460 self.profileIndex
461 461 self.flagDiscontinuousBlock
462 462 self.flagIsNewBlock
463 463 """
464 464 if self.flagNoMoreFiles:
465 465 self.dataOut.flagNoData = True
466 466 print('Process finished')
467 467 return 0
468 468 self.flagDiscontinuousBlock = 0
469 469 self.flagIsNewBlock = 0
470 470 if self.__hasNotDataInBuffer():
471 471 if not(self.readNextBlock()):
472 472 return 0
473 473
474 474 self.getFirstHeader()
475 475
476 476 self.reshapeData()
477 477 if self.datablock is None:
478 478 self.dataOut.flagNoData = True
479 479 return 0
480 480
481 481 if not self.getByBlock:
482 482
483 483 """
484 484 Return profile by profile
485 485
486 486 If nTxs > 1 then one profile is divided by nTxs and number of total
487 487 blocks is increased by nTxs (nProfiles *= nTxs)
488 488 """
489 489 self.dataOut.flagDataAsBlock = False
490 490 self.dataOut.data = self.datablock[:, self.profileIndex, :]
491 491 self.dataOut.profileIndex = self.profileIndex
492 492
493 493 self.profileIndex += 1
494 494
495 495 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
496 496 # """
497 497 # Return all block
498 498 # """
499 499 # self.dataOut.flagDataAsBlock = True
500 500 # self.dataOut.data = self.datablock
501 501 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
502 502 #
503 503 # self.profileIndex = self.dataOut.nProfiles
504 504
505 505 else:
506 506 """
507 507 Return a block
508 508 """
509 509 if self.selBlocksize == None:
510 510 self.selBlocksize = self.dataOut.nProfiles
511 511 if self.selBlocktime != None:
512 512 if self.dataOut.nCohInt is not None:
513 513 nCohInt = self.dataOut.nCohInt
514 514 else:
515 515 nCohInt = 1
516 516 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
517 517 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
518 518
519 519 self.dataOut.data = self.datablock[:,
520 520 self.profileIndex:self.profileIndex + self.selBlocksize, :]
521 521 self.profileIndex += self.selBlocksize
522 522 datasize = self.dataOut.data.shape[1]
523 523
524 524 if datasize < self.selBlocksize:
525 525 buffer = numpy.zeros(
526 526 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
527 527 buffer[:, :datasize, :] = self.dataOut.data
528 528
529 529 while datasize < self.selBlocksize: # Not enough profiles to fill the block
530 530 if not(self.readNextBlock()):
531 531 return 0
532 532 self.getFirstHeader()
533 533 self.reshapeData()
534 534 if self.datablock is None:
535 535 self.dataOut.flagNoData = True
536 536 return 0
537 537 # stack data
538 538 blockIndex = self.selBlocksize - datasize
539 539 datablock1 = self.datablock[:, :blockIndex, :]
540 540
541 541 buffer[:, datasize:datasize +
542 542 datablock1.shape[1], :] = datablock1
543 543 datasize += datablock1.shape[1]
544 544
545 545 self.dataOut.data = buffer
546 546 self.profileIndex = blockIndex
547 547
548 548 self.dataOut.flagDataAsBlock = True
549 549 self.dataOut.nProfiles = self.dataOut.data.shape[1]
550 550
551 551 self.dataOut.flagNoData = False
552 552
553 553 self.getBasicHeader()
554 554
555 555 self.dataOut.realtime = self.online
556 556
557 557 return self.dataOut.data
558 558
559 559
560 560 class VoltageWriter(JRODataWriter, Operation):
561 561 """
562 562 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
563 563 de los datos siempre se realiza por bloques.
564 564 """
565 565
566 566 ext = ".r"
567 567
568 568 optchar = "D"
569 569
570 570 shapeBuffer = None
571 571
572 572 def __init__(self, **kwargs):
573 573 """
574 574 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
575 575
576 576 Affected:
577 577 self.dataOut
578 578
579 579 Return: None
580 580 """
581 581 Operation.__init__(self, **kwargs)
582 582
583 583 self.nTotalBlocks = 0
584 584
585 585 self.profileIndex = 0
586 586
587 587 self.isConfig = False
588 588
589 589 self.fp = None
590 590
591 591 self.flagIsNewFile = 1
592 592
593 593 self.blockIndex = 0
594 594
595 595 self.flagIsNewBlock = 0
596 596
597 597 self.setFile = None
598 598
599 599 self.dtype = None
600 600
601 601 self.path = None
602 602
603 603 self.filename = None
604 604
605 605 self.basicHeaderObj = BasicHeader(LOCALTIME)
606 606
607 607 self.systemHeaderObj = SystemHeader()
608 608
609 609 self.radarControllerHeaderObj = RadarControllerHeader()
610 610
611 611 self.processingHeaderObj = ProcessingHeader()
612 612
613 613 def hasAllDataInBuffer(self):
614 614 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
615 615 return 1
616 616 return 0
617 617
618 618 def setBlockDimension(self):
619 619 """
620 620 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
621 621
622 622 Affected:
623 623 self.shape_spc_Buffer
624 624 self.shape_cspc_Buffer
625 625 self.shape_dc_Buffer
626 626
627 627 Return: None
628 628 """
629 629 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
630 630 self.processingHeaderObj.nHeights,
631 631 self.systemHeaderObj.nChannels)
632 632
633 633 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
634 634 self.processingHeaderObj.profilesPerBlock,
635 635 self.processingHeaderObj.nHeights),
636 636 dtype=numpy.dtype('complex64'))
637 637
638 638 def writeBlock(self):
639 639 """
640 640 Escribe el buffer en el file designado
641 641
642 642 Affected:
643 643 self.profileIndex
644 644 self.flagIsNewFile
645 645 self.flagIsNewBlock
646 646 self.nTotalBlocks
647 647 self.blockIndex
648 648
649 649 Return: None
650 650 """
651 651 data = numpy.zeros(self.shapeBuffer, self.dtype)
652 652
653 653 junk = numpy.transpose(self.datablock, (1, 2, 0))
654 654
655 655 data['real'] = junk.real
656 656 data['imag'] = junk.imag
657 657
658 658 data = data.reshape((-1))
659 659
660 660 data.tofile(self.fp)
661 661
662 662 self.datablock.fill(0)
663 663
664 664 self.profileIndex = 0
665 665 self.flagIsNewFile = 0
666 666 self.flagIsNewBlock = 1
667 667
668 668 self.blockIndex += 1
669 669 self.nTotalBlocks += 1
670 670
671 671 # print "[Writing] Block = %04d" %self.blockIndex
672 672
673 673 def putData(self):
674 674 """
675 675 Setea un bloque de datos y luego los escribe en un file
676 676
677 677 Affected:
678 678 self.flagIsNewBlock
679 679 self.profileIndex
680 680
681 681 Return:
682 682 0 : Si no hay data o no hay mas files que puedan escribirse
683 683 1 : Si se escribio la data de un bloque en un file
684 684 """
685 685 if self.dataOut.flagNoData:
686 686 return 0
687 687
688 688 self.flagIsNewBlock = 0
689 689
690 690 if self.dataOut.flagDiscontinuousBlock:
691 691 self.datablock.fill(0)
692 692 self.profileIndex = 0
693 693 self.setNextFile()
694 694
695 695 if self.profileIndex == 0:
696 696 self.setBasicHeader()
697 697
698 698 self.datablock[:, self.profileIndex, :] = self.dataOut.data
699 699
700 700 self.profileIndex += 1
701 701
702 702 if self.hasAllDataInBuffer():
703 703 # if self.flagIsNewFile:
704 704 self.writeNextBlock()
705 705 # self.setFirstHeader()
706 706
707 707 return 1
708 708
709 709 def __getBlockSize(self):
710 710 '''
711 711 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
712 712 '''
713 713
714 714 dtype_width = self.getDtypeWidth()
715 715
716 716 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
717 717 self.profilesPerBlock * dtype_width * 2)
718 718
719 719 return blocksize
720 720
721 721 def setFirstHeader(self):
722 722 """
723 723 Obtiene una copia del First Header
724 724
725 725 Affected:
726 726 self.systemHeaderObj
727 727 self.radarControllerHeaderObj
728 728 self.dtype
729 729
730 730 Return:
731 731 None
732 732 """
733 733
734 734 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
735 735 self.systemHeaderObj.nChannels = self.dataOut.nChannels
736 736 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
737 737
738 738 self.processingHeaderObj.dtype = 0 # Voltage
739 739 self.processingHeaderObj.blockSize = self.__getBlockSize()
740 740 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
741 741 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
742 742 # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
743 743 self.processingHeaderObj.nWindows = 1
744 744 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
745 745 # Cuando la data de origen es de tipo Voltage
746 746 self.processingHeaderObj.nIncohInt = 1
747 747 # Cuando la data de origen es de tipo Voltage
748 748 self.processingHeaderObj.totalSpectra = 0
749 749
750 750 if self.dataOut.code is not None:
751 751 self.processingHeaderObj.code = self.dataOut.code
752 752 self.processingHeaderObj.nCode = self.dataOut.nCode
753 753 self.processingHeaderObj.nBaud = self.dataOut.nBaud
754 754
755 755 if self.processingHeaderObj.nWindows != 0:
756 756 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
757 757 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
758 758 self.dataOut.heightList[0]
759 759 self.processingHeaderObj.nHeights = self.dataOut.nHeights
760 760 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
761 761
762 762 self.processingHeaderObj.processFlags = self.getProcessFlags()
763 763
764 self.setBasicHeader() No newline at end of file
764 self.setBasicHeader()
765 No newline at end of file
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (606 lines changed) Show them Hide them
@@ -1,360 +1,538
1 1 '''
2
3 $Author: murco $
4 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
2 Updated for multiprocessing
3 Author : Sergio Cortez
4 Jan 2018
5 Abstract:
6 Base class for processing units and operations. A decorator provides multiprocessing features and interconnect the processes created.
7 The argument (kwargs) sent from the controller is parsed and filtered via the decorator for each processing unit or operation instantiated.
8 The decorator handle also the methods inside the processing unit to be called from the main script (not as operations) (OPERATION -> type ='self').
9
10 Based on:
11 $Author: murco $
12 $Id: jroproc_base.py 1 2012-11-12 18:56:07Z murco $
5 13 '''
14 from platform import python_version
6 15 import inspect
7 from fuzzywuzzy import process
8
9 def checkKwargs(method, kwargs):
10 currentKwargs = kwargs
11 choices = inspect.getargspec(method).args
12 try:
13 choices.remove('self')
14 except Exception as e:
15 pass
16 import zmq
17 import time
18 import pickle
19 import os
20 from multiprocessing import Process
16 21
17 try:
18 choices.remove('dataOut')
19 except Exception as e:
20 pass
22 from schainpy.utils import log
21 23
22 for kwarg in kwargs:
23 fuzz = process.extractOne(kwarg, choices)
24 if fuzz is None:
25 continue
26 if fuzz[1] < 100:
27 raise Exception('\x1b[0;32;40mDid you mean {} instead of {} in {}? \x1b[0m'.
28 format(fuzz[0], kwarg, method.__self__.__class__.__name__))
29 24
30 25 class ProcessingUnit(object):
31 26
32 27 """
33 Esta es la clase base para el procesamiento de datos.
28 Update - Jan 2018 - MULTIPROCESSING
29 All the "call" methods present in the previous base were removed.
30 The majority of operations are independant processes, thus
31 the decorator is in charge of communicate the operation processes
32 with the proccessing unit via IPC.
34 33
35 Contiene el metodo "call" para llamar operaciones. Las operaciones pueden ser:
36 - Metodos internos (callMethod)
37 - Objetos del tipo Operation (callObject). Antes de ser llamados, estos objetos
38 tienen que ser agreagados con el metodo "add".
34 The constructor does not receive any argument. The remaining methods
35 are related with the operations to execute.
36
39 37
40 38 """
41 39 # objeto de datos de entrada (Voltage, Spectra o Correlation)
42 40 dataIn = None
43 41 dataInList = []
44 42
45 43 # objeto de datos de entrada (Voltage, Spectra o Correlation)
44
45 id = None
46 inputId = None
47
46 48 dataOut = None
47 49
50 dictProcs = None
51
48 52 operations2RunDict = None
49 53
50 54 isConfig = False
51 55
52
53 def __init__(self, *args, **kwargs):
56 def __init__(self):
54 57
55 58 self.dataIn = None
56 self.dataInList = []
57
58 59 self.dataOut = None
59 60
60 self.operations2RunDict = {}
61 self.operationKwargs = {}
62
63 61 self.isConfig = False
64 62
65 self.args = args
66 self.kwargs = kwargs
67
68 if not hasattr(self, 'name'):
69 self.name = self.__class__.__name__
70
71 checkKwargs(self.run, kwargs)
72
73 63 def getAllowedArgs(self):
74 64 if hasattr(self, '__attrs__'):
75 65 return self.__attrs__
76 66 else:
77 67 return inspect.getargspec(self.run).args
78 68
79 69 def addOperationKwargs(self, objId, **kwargs):
80 70 '''
81 71 '''
82 72
83 73 self.operationKwargs[objId] = kwargs
84
85
74
86 75 def addOperation(self, opObj, objId):
87 76
88 77 """
89 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
90 identificador asociado a este objeto.
78 This method is used in the controller, and update the dictionary containing the operations to execute. The dict
79 posses the id of the operation process (IPC purposes)
91 80
92 Input:
81 Agrega un objeto del tipo "Operation" (opObj) a la lista de objetos "self.objectList" y retorna el
82 identificador asociado a este objeto.
93 83
94 object : objeto de la clase "Operation"
84 Input:
95 85
96 Return:
86 object : objeto de la clase "Operation"
87
88 Return:
97 89
98 objId : identificador del objeto, necesario para ejecutar la operacion
90 objId : identificador del objeto, necesario para comunicar con master(procUnit)
99 91 """
100 92
101 93 self.operations2RunDict[objId] = opObj
102 94
103 95 return objId
104 96
97
105 98 def getOperationObj(self, objId):
106 99
107 100 if objId not in list(self.operations2RunDict.keys()):
108 101 return None
109 102
110 103 return self.operations2RunDict[objId]
111 104
112 105 def operation(self, **kwargs):
113 106
114 107 """
115 108 Operacion directa sobre la data (dataOut.data). Es necesario actualizar los valores de los
116 109 atributos del objeto dataOut
117 110
118 111 Input:
119 112
120 113 **kwargs : Diccionario de argumentos de la funcion a ejecutar
121 114 """
122 115
123 raise NotImplementedError
124
125 def callMethod(self, name, opId):
126
127 """
128 Ejecuta el metodo con el nombre "name" y con argumentos **kwargs de la propia clase.
129
130 Input:
131 name : nombre del metodo a ejecutar
132
133 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
134
135 """
116 raise NotImplementedError
136 117
137 #Checking the inputs
138 if name == 'run':
139
140 if not self.checkInputs():
141 self.dataOut.flagNoData = True
142 return False
143 else:
144 #Si no es un metodo RUN la entrada es la misma dataOut (interna)
145 if self.dataOut is not None and self.dataOut.isEmpty():
146 return False
118 def setup(self):
147 119
148 #Getting the pointer to method
149 methodToCall = getattr(self, name)
120 raise NotImplementedError
150 121
151 #Executing the self method
122 def run(self):
152 123
153 if hasattr(self, 'mp'):
154 if name=='run':
155 if self.mp is False:
156 self.mp = True
157 self.start()
158 else:
159 self.operationKwargs[opId]['parent'] = self.kwargs
160 methodToCall(**self.operationKwargs[opId])
161 else:
162 if name=='run':
163 methodToCall(**self.kwargs)
164 else:
165 methodToCall(**self.operationKwargs[opId])
124 raise NotImplementedError
166 125
167 if self.dataOut is None:
168 return False
126 def close(self):
127 #Close every thread, queue or any other object here is it is neccesary.
128 return
129
130 class Operation(object):
169 131
170 if self.dataOut.isEmpty():
171 return False
132 """
133 Update - Jan 2018 - MULTIPROCESSING
172 134
173 return True
135 Most of the methods remained the same. The decorator parse the arguments and executed the run() method for each process.
136 The constructor doe snot receive any argument, neither the baseclass.
174 137
175 def callObject(self, objId):
176 138
177 """
178 Ejecuta la operacion asociada al identificador del objeto "objId"
139 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
140 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
141 acumulacion dentro de esta clase
179 142
180 Input:
143 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
181 144
182 objId : identificador del objeto a ejecutar
145 """
146 id = None
147 __buffer = None
148 dest = None
149 isConfig = False
150 readyFlag = None
183 151
184 **kwargs : diccionario con los nombres y valores de la funcion a ejecutar.
152 def __init__(self):
185 153
186 Return:
154 self.buffer = None
155 self.dest = None
156 self.isConfig = False
157 self.readyFlag = False
187 158
188 None
189 """
159 if not hasattr(self, 'name'):
160 self.name = self.__class__.__name__
161
162 def getAllowedArgs(self):
163 if hasattr(self, '__attrs__'):
164 return self.__attrs__
165 else:
166 return inspect.getargspec(self.run).args
190 167
191 if self.dataOut is not None and self.dataOut.isEmpty():
192 return False
168 def setup(self):
193 169
194 externalProcObj = self.operations2RunDict[objId]
170 self.isConfig = True
195 171
196 if hasattr(externalProcObj, 'mp'):
197 if externalProcObj.mp is False:
198 externalProcObj.kwargs['parent'] = self.kwargs
199 self.operationKwargs[objId] = externalProcObj.kwargs
200 externalProcObj.mp = True
201 externalProcObj.start()
202 else:
203 externalProcObj.run(self.dataOut, **externalProcObj.kwargs)
204 self.operationKwargs[objId] = externalProcObj.kwargs
172 raise NotImplementedError
205 173
206 174
207 return True
175 def run(self, dataIn, **kwargs):
208 176
209 def call(self, opType, opName=None, opId=None):
210 177 """
211 Return True si ejecuta la operacion interna nombrada "opName" o la operacion externa
212 identificada con el id "opId"; con los argumentos "**kwargs".
213
214 False si la operacion no se ha ejecutado.
178 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
179 atributos del objeto dataIn.
215 180
216 181 Input:
217 182
218 opType : Puede ser "self" o "external"
183 dataIn : objeto del tipo JROData
219 184
220 Depende del tipo de operacion para llamar a:callMethod or callObject:
185 Return:
221 186
222 1. If opType = "self": Llama a un metodo propio de esta clase:
187 None
223 188
224 name_method = getattr(self, name)
225 name_method(**kwargs)
189 Affected:
190 __buffer : buffer de recepcion de datos.
226 191
192 """
193 if not self.isConfig:
194 self.setup(**kwargs)
227 195
228 2. If opType = "other" o"external": Llama al metodo "run()" de una instancia de la
229 clase "Operation" o de un derivado de ella:
196 raise NotImplementedError
230 197
231 instanceName = self.operationList[opId]
232 instanceName.run(**kwargs)
198 def close(self):
233 199
234 opName : Si la operacion es interna (opType = 'self'), entonces el "opName" sera
235 usada para llamar a un metodo interno de la clase Processing
200 pass
236 201
237 opId : Si la operacion es externa (opType = 'other' o 'external), entonces el
238 "opId" sera usada para llamar al metodo "run" de la clase Operation
239 registrada anteriormente con ese Id
240 202
241 Exception:
242 Este objeto de tipo Operation debe de haber sido agregado antes con el metodo:
243 "addOperation" e identificado con el valor "opId" = el id de la operacion.
244 De lo contrario retornara un error del tipo ValueError
203 ######### Decorator #########
245 204
246 """
247 205
248 if opType == 'self':
206 def MPDecorator(BaseClass):
207
208 """
209 "Multiprocessing class decorator"
249 210
250 if not opName:
251 raise ValueError("opName parameter should be defined")
211 This function add multiprocessing features to the base class. Also,
212 it handle the communication beetween processes (readers, procUnits and operations).
213 Receive the arguments at the moment of instantiation. According to that, discriminates if it
214 is a procUnit or an operation
215 """
216
217 class MPClass(BaseClass, Process):
218
219 "This is the overwritten class"
220 operations2RunDict = None
221 socket_l = None
222 socket_p = None
223 socketOP = None
224 socket_router = None
225 dictProcs = None
226 typeProc = None
227 def __init__(self, *args, **kwargs):
228 super(MPClass, self).__init__()
229 Process.__init__(self)
230
231
232 self.operationKwargs = {}
233 self.args = args
234
235
236 self.operations2RunDict = {}
237 self.kwargs = kwargs
238
239 # The number of arguments (args) determine the type of process
240
241 if len(self.args) is 3:
242 self.typeProc = "ProcUnit"
243 self.id = args[0] #topico de publicacion
244 self.inputId = args[1] #topico de subcripcion
245 self.dictProcs = args[2] #diccionario de procesos globales
246 else:
247 self.id = args[0]
248 self.typeProc = "Operation"
249
250 def addOperationKwargs(self, objId, **kwargs):
251
252 self.operationKwargs[objId] = kwargs
252 253
253 sts = self.callMethod(opName, opId)
254 def getAllowedArgs(self):
254 255
255 elif opType == 'other' or opType == 'external' or opType == 'plotter':
256 if hasattr(self, '__attrs__'):
257 return self.__attrs__
258 else:
259 return inspect.getargspec(self.run).args
260
261
262 def sockListening(self, topic):
263
264 """
265 This function create a socket to receive objects.
266 The 'topic' argument is related to the publisher process from which the self process is
267 listening (data).
268 In the case were the self process is listening to a Reader (proc Unit),
269 special conditions are introduced to maximize parallelism.
270 """
271
272 cont = zmq.Context()
273 zmq_socket = cont.socket(zmq.SUB)
274 if not os.path.exists('/tmp/socketTmp'):
275 os.mkdir('/tmp/socketTmp')
276
277 if 'Reader' in self.dictProcs[self.inputId].name:
278 zmq_socket.connect('ipc:///tmp/socketTmp/b')
279
280 else:
281 zmq_socket.connect('ipc:///tmp/socketTmp/%s' % self.inputId)
282
283 #log.error('RECEIVING FROM {} {}'.format(self.inputId, str(topic).encode()))
284 zmq_socket.setsockopt(zmq.SUBSCRIBE, str(topic).encode()) #yong
256 285
257 if not opId:
258 raise ValueError("opId parameter should be defined")
286 return zmq_socket
259 287
260 if opId not in list(self.operations2RunDict.keys()):
261 raise ValueError("Any operation with id=%s has been added" %str(opId))
262 288
263 sts = self.callObject(opId)
289 def listenProc(self, sock):
264 290
265 else:
266 raise ValueError("opType should be 'self', 'external' or 'plotter'; and not '%s'" %opType)
291 """
292 This function listen to a ipc addres until a message is recovered. To serialize the
293 data (object), pickle has been use.
294 The 'sock' argument is the socket previously connect to an ipc address and with a topic subscription.
295 """
296
297 a = sock.recv_multipart()
298 a = pickle.loads(a[1])
299 return a
267 300
268 return sts
301 def sockPublishing(self):
269 302
270 def setInput(self, dataIn):
303 """
304 This function create a socket for publishing purposes.
305 Depending on the process type from where is created, it binds or connect
306 to special IPC addresses.
307 """
308 time.sleep(4) #yong
309 context = zmq.Context()
310 zmq_socket = context.socket(zmq.PUB)
311 if not os.path.exists('/tmp/socketTmp'): os.mkdir('/tmp/socketTmp')
312 if 'Reader' in self.dictProcs[self.id].name:
313 zmq_socket.connect('ipc:///tmp/socketTmp/a')
314 else:
315 zmq_socket.bind('ipc:///tmp/socketTmp/%s' % self.id)
271 316
272 self.dataIn = dataIn
273 self.dataInList.append(dataIn)
317 return zmq_socket
274 318
275 def getOutputObj(self):
319 def publishProc(self, sock, data):
276 320
277 return self.dataOut
321 """
322 This function publish a python object (data) under a specific topic in a socket (sock).
323 Usually, the topic is the self id of the process.
324 """
278 325
279 def checkInputs(self):
326 sock.send_multipart([str(self.id).encode(), pickle.dumps(data)]) #yong
327
328 return True
280 329
281 for thisDataIn in self.dataInList:
330 def sockOp(self):
282 331
283 if thisDataIn.isEmpty():
284 return False
332 """
333 This function create a socket for communication purposes with operation processes.
334 """
285 335
286 return True
336 cont = zmq.Context()
337 zmq_socket = cont.socket(zmq.DEALER)
338
339 if python_version()[0] == '2':
340 zmq_socket.setsockopt(zmq.IDENTITY, self.id)
341 if python_version()[0] == '3':
342 zmq_socket.setsockopt_string(zmq.IDENTITY, self.id)
287 343
288 def setup(self):
289 344
290 raise NotImplementedError
345 return zmq_socket
291 346
292 def run(self):
293 347
294 raise NotImplementedError
348 def execOp(self, socket, opId, dataObj):
295 349
296 def close(self):
297 #Close every thread, queue or any other object here is it is neccesary.
298 return
350 """
351 This function 'execute' an operation main routine by establishing a
352 connection with it and sending a python object (dataOut).
353 """
354 if not os.path.exists('/tmp/socketTmp'): os.mkdir('/tmp/socketTmp')
355 socket.connect('ipc:///tmp/socketTmp/%s' %opId)
356
357
358 socket.send(pickle.dumps(dataObj)) #yong
359
360 argument = socket.recv_multipart()[0]
361
362 argument = pickle.loads(argument)
363
364 return argument
365
366 def sockIO(self):
299 367
300 class Operation(object):
368 """
369 Socket defined for an operation process. It is able to recover the object sent from another process as well as a
370 identifier of who sent it.
371 """
301 372
302 """
303 Clase base para definir las operaciones adicionales que se pueden agregar a la clase ProcessingUnit
304 y necesiten acumular informacion previa de los datos a procesar. De preferencia usar un buffer de
305 acumulacion dentro de esta clase
373 cont = zmq.Context()
374 if not os.path.exists('/tmp/socketTmp'): os.mkdir('/tmp/socketTmp')
375 socket = cont.socket(zmq.ROUTER)
376 socket.bind('ipc:///tmp/socketTmp/%s' % self.id)
306 377
307 Ejemplo: Integraciones coherentes, necesita la informacion previa de los n perfiles anteriores (bufffer)
378 return socket
308 379
309 """
380 def funIOrec(self, socket):
310 381
311 __buffer = None
312 isConfig = False
382 """
383 Operation method, recover the id of the process who sent a python object.
384 The 'socket' argument is the socket binded to a specific process ipc.
385 """
313 386
314 def __init__(self, **kwargs):
387 #id_proc = socket.recv()
388
389 #dataObj = socket.recv_pyobj()
390
391 dataObj = socket.recv_multipart()
392
393 dataObj[1] = pickle.loads(dataObj[1])
394 return dataObj[0], dataObj[1]
395
396 def funIOsen(self, socket, data, dest):
397
398 """
399 Operation method, send a python object to a specific destination.
400 The 'dest' argument is the id of a proccesinf unit.
401 """
402
403 socket.send_multipart([dest, pickle.dumps(data)]) #yong
315 404
316 self.__buffer = None
317 self.isConfig = False
318 self.kwargs = kwargs
319 if not hasattr(self, 'name'):
320 self.name = self.__class__.__name__
321 checkKwargs(self.run, kwargs)
405 return True
322 406
323 def getAllowedArgs(self):
324 if hasattr(self, '__attrs__'):
325 return self.__attrs__
326 else:
327 return inspect.getargspec(self.run).args
328 407
329 def setup(self):
408 def runReader(self):
330 409
331 self.isConfig = True
410 # time.sleep(3)
411 while True:
412
413 BaseClass.run(self, **self.kwargs)
332 414
333 raise NotImplementedError
334 415
335 def run(self, dataIn, **kwargs):
416 keyList = list(self.operations2RunDict.keys())
417 keyList.sort()
418
419 for key in keyList:
420 self.socketOP = self.sockOp()
421 self.dataOut = self.execOp(self.socketOP, key, self.dataOut)
336 422
337 """
338 Realiza las operaciones necesarias sobre la dataIn.data y actualiza los
339 atributos del objeto dataIn.
423
424 if self.flagNoMoreFiles: #Usar un objeto con flags para saber si termino el proc o hubo un error
425 self.publishProc(self.socket_p, "Finish")
426 break
340 427
341 Input:
342
343 dataIn : objeto del tipo JROData
428 if self.dataOut.flagNoData:
429 continue
430
431 #print("Publishing data...")
432 self.publishProc(self.socket_p, self.dataOut)
433 # time.sleep(2)
434
435
436 print("%s done" %BaseClass.__name__)
437 return 0
438
439 def runProc(self):
344 440
345 Return:
441 # All the procUnits with kwargs that require a setup initialization must be defined here.
346 442
347 None
443 if self.setupReq:
444 BaseClass.setup(self, **self.kwargs)
348 445
349 Affected:
350 __buffer : buffer de recepcion de datos.
446 while True:
447 self.dataIn = self.listenProc(self.socket_l)
448 #print("%s received data" %BaseClass.__name__)
449
450 if self.dataIn == "Finish":
451 break
452
453 m_arg = list(self.kwargs.keys())
454 num_arg = list(range(1,int(BaseClass.run.__code__.co_argcount)))
455
456 run_arg = {}
457
458 for var in num_arg:
459 if BaseClass.run.__code__.co_varnames[var] in m_arg:
460 run_arg[BaseClass.run.__code__.co_varnames[var]] = self.kwargs[BaseClass.run.__code__.co_varnames[var]]
461
462 #BaseClass.run(self, **self.kwargs)
463 BaseClass.run(self, **run_arg)
464
465 ## Iterar sobre una serie de data que podrias aplicarse
466
467 for m_name in BaseClass.METHODS:
468
469 met_arg = {}
470
471 for arg in m_arg:
472 if arg in BaseClass.METHODS[m_name]:
473 for att in BaseClass.METHODS[m_name]:
474 met_arg[att] = self.kwargs[att]
475
476 method = getattr(BaseClass, m_name)
477 method(self, **met_arg)
478 break
479
480 if self.dataOut.flagNoData:
481 continue
482
483 keyList = list(self.operations2RunDict.keys())
484 keyList.sort()
485
486 for key in keyList:
487
488 self.socketOP = self.sockOp()
489 self.dataOut = self.execOp(self.socketOP, key, self.dataOut)
490
491
492 self.publishProc(self.socket_p, self.dataOut)
493
494
495 print("%s done" %BaseClass.__name__)
496
497 return 0
498
499 def runOp(self):
500
501 while True:
502
503 [self.dest ,self.buffer] = self.funIOrec(self.socket_router)
504
505 self.buffer = BaseClass.run(self, self.buffer, **self.kwargs)
506
507 self.funIOsen(self.socket_router, self.buffer, self.dest)
508
509 print("%s done" %BaseClass.__name__)
510 return 0
511
512
513 def run(self):
514
515 if self.typeProc is "ProcUnit":
516
517 self.socket_p = self.sockPublishing()
518
519 if 'Reader' not in self.dictProcs[self.id].name:
520 self.socket_l = self.sockListening(self.inputId)
521 self.runProc()
522
523 else:
524
525 self.runReader()
526
527 elif self.typeProc is "Operation":
528
529 self.socket_router = self.sockIO()
530
531 self.runOp()
351 532
352 """
353 if not self.isConfig:
354 self.setup(**kwargs)
355
356 raise NotImplementedError
357
358 def close(self):
533 else:
534 raise ValueError("Unknown type")
359 535
360 pass No newline at end of file
536 return 0
537
538 return MPClass No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,953 +1,966
1 1 import itertools
2 2
3 3 import numpy
4 4
5 from .jroproc_base import ProcessingUnit, Operation
5 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
6 6 from schainpy.model.data.jrodata import Spectra
7 7 from schainpy.model.data.jrodata import hildebrand_sekhon
8 from schainpy.utils import log #yong
8 from schainpy.utils import log
9 9
10 @MPDecorator
10 11 class SpectraProc(ProcessingUnit):
11 12
12 def __init__(self, **kwargs):
13 METHODS = {'selectHeights' : ['minHei', 'maxHei'],
14 'selectChannels' : 'channelList',
15 'selectChannelsByIndex': 'channelIndexList',
16 'getBeaconSignal' : ['tauindex', 'channelindex', 'hei_ref'],
17 'selectHeightsByIndex' : ['minIndex', 'maxIndex']
18 }
13 19
14 ProcessingUnit.__init__(self, **kwargs)
20 def __init__(self):#, **kwargs):
21
22 ProcessingUnit.__init__(self)#, **kwargs)
15 23
16 24 self.buffer = None
17 25 self.firstdatatime = None
18 26 self.profIndex = 0
19 27 self.dataOut = Spectra()
20 28 self.id_min = None
21 29 self.id_max = None
30 self.setupReq = False #Agregar a todas las unidades de proc
22 31
23 32 def __updateSpecFromVoltage(self):
24 33
25 34 self.dataOut.timeZone = self.dataIn.timeZone
26 35 self.dataOut.dstFlag = self.dataIn.dstFlag
27 36 self.dataOut.errorCount = self.dataIn.errorCount
28 37 self.dataOut.useLocalTime = self.dataIn.useLocalTime
29 38 try:
30 39 self.dataOut.processingHeaderObj = self.dataIn.processingHeaderObj.copy()
31 40 except:
32 41 pass
33 42 self.dataOut.radarControllerHeaderObj = self.dataIn.radarControllerHeaderObj.copy()
34 43 self.dataOut.systemHeaderObj = self.dataIn.systemHeaderObj.copy()
35 44 self.dataOut.channelList = self.dataIn.channelList
36 45 self.dataOut.heightList = self.dataIn.heightList
37 46 self.dataOut.dtype = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
38 47
39 48 self.dataOut.nBaud = self.dataIn.nBaud
40 49 self.dataOut.nCode = self.dataIn.nCode
41 50 self.dataOut.code = self.dataIn.code
42 51 self.dataOut.nProfiles = self.dataOut.nFFTPoints
43 52
44 53 self.dataOut.flagDiscontinuousBlock = self.dataIn.flagDiscontinuousBlock
45 54 self.dataOut.utctime = self.firstdatatime
46 55 # asumo q la data esta decodificada
47 56 self.dataOut.flagDecodeData = self.dataIn.flagDecodeData
48 57 # asumo q la data esta sin flip
49 58 self.dataOut.flagDeflipData = self.dataIn.flagDeflipData
50 59 self.dataOut.flagShiftFFT = False
51 60
52 61 self.dataOut.nCohInt = self.dataIn.nCohInt
53 62 self.dataOut.nIncohInt = 1
54 63
55 64 self.dataOut.windowOfFilter = self.dataIn.windowOfFilter
56 65
57 66 self.dataOut.frequency = self.dataIn.frequency
58 67 self.dataOut.realtime = self.dataIn.realtime
59 68
60 69 self.dataOut.azimuth = self.dataIn.azimuth
61 70 self.dataOut.zenith = self.dataIn.zenith
62 71
63 72 self.dataOut.beam.codeList = self.dataIn.beam.codeList
64 73 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
65 74 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
66 75
67 76 def __getFft(self):
68 77 """
69 78 Convierte valores de Voltaje a Spectra
70 79
71 80 Affected:
72 81 self.dataOut.data_spc
73 82 self.dataOut.data_cspc
74 83 self.dataOut.data_dc
75 84 self.dataOut.heightList
76 85 self.profIndex
77 86 self.buffer
78 87 self.dataOut.flagNoData
79 88 """
80 89 fft_volt = numpy.fft.fft(
81 90 self.buffer, n=self.dataOut.nFFTPoints, axis=1)
82 91 fft_volt = fft_volt.astype(numpy.dtype('complex'))
83 92 dc = fft_volt[:, 0, :]
84 93
85 94 # calculo de self-spectra
86 95 fft_volt = numpy.fft.fftshift(fft_volt, axes=(1,))
87 96 spc = fft_volt * numpy.conjugate(fft_volt)
88 97 spc = spc.real
89 98
90 99 blocksize = 0
91 100 blocksize += dc.size
92 101 blocksize += spc.size
93 102
94 103 cspc = None
95 104 pairIndex = 0
96 105 if self.dataOut.pairsList != None:
97 106 # calculo de cross-spectra
98 107 cspc = numpy.zeros(
99 108 (self.dataOut.nPairs, self.dataOut.nFFTPoints, self.dataOut.nHeights), dtype='complex')
100 109 for pair in self.dataOut.pairsList:
101 110 if pair[0] not in self.dataOut.channelList:
102 111 raise ValueError("Error getting CrossSpectra: pair 0 of %s is not in channelList = %s" % (
103 112 str(pair), str(self.dataOut.channelList)))
104 113 if pair[1] not in self.dataOut.channelList:
105 114 raise ValueError("Error getting CrossSpectra: pair 1 of %s is not in channelList = %s" % (
106 115 str(pair), str(self.dataOut.channelList)))
107 116
108 117 cspc[pairIndex, :, :] = fft_volt[pair[0], :, :] * \
109 118 numpy.conjugate(fft_volt[pair[1], :, :])
110 119 pairIndex += 1
111 120 blocksize += cspc.size
112 121
113 122 self.dataOut.data_spc = spc
114 123 self.dataOut.data_cspc = cspc
115 124 self.dataOut.data_dc = dc
116 125 self.dataOut.blockSize = blocksize
117 126 self.dataOut.flagShiftFFT = True
118 127
119 128 def run(self, nProfiles=None, nFFTPoints=None, pairsList=[], ippFactor=None, shift_fft=False):
120 129
121 130 self.dataOut.flagNoData = True
122 131
123 132 if self.dataIn.type == "Spectra":
124 133 self.dataOut.copy(self.dataIn)
125 134 # if not pairsList:
126 135 # pairsList = itertools.combinations(self.dataOut.channelList, 2)
127 136 # if self.dataOut.data_cspc is not None:
128 137 # self.__selectPairs(pairsList)
129 138 if shift_fft:
130 139 #desplaza a la derecha en el eje 2 determinadas posiciones
131 140 shift = int(self.dataOut.nFFTPoints/2)
132 141 self.dataOut.data_spc = numpy.roll(self.dataOut.data_spc, shift , axis=1)
133 142
134 143 if self.dataOut.data_cspc is not None:
135 144 #desplaza a la derecha en el eje 2 determinadas posiciones
136 145 self.dataOut.data_cspc = numpy.roll(self.dataOut.data_cspc, shift, axis=1)
137
146
138 147 return True
139 148
140 149 if self.dataIn.type == "Voltage":
141 150
142 151 if nFFTPoints == None:
143 152 raise ValueError("This SpectraProc.run() need nFFTPoints input variable")
144 153
145 154 if nProfiles == None:
146 155 nProfiles = nFFTPoints
147 156
148 157 if ippFactor == None:
149 158 ippFactor = 1
150 159
151 160 self.dataOut.ippFactor = ippFactor
152 161
153 162 self.dataOut.nFFTPoints = nFFTPoints
154 163 self.dataOut.pairsList = pairsList
155 164
156 165 if self.buffer is None:
157 166 self.buffer = numpy.zeros((self.dataIn.nChannels,
158 167 nProfiles,
159 168 self.dataIn.nHeights),
160 169 dtype='complex')
161 170
162 171 if self.dataIn.flagDataAsBlock:
163 172 # data dimension: [nChannels, nProfiles, nSamples]
164 173 nVoltProfiles = self.dataIn.data.shape[1]
165 174 # nVoltProfiles = self.dataIn.nProfiles
166 175
167 176 if nVoltProfiles == nProfiles:
168 177 self.buffer = self.dataIn.data.copy()
169 178 self.profIndex = nVoltProfiles
170 179
171 180 elif nVoltProfiles < nProfiles:
172 181
173 182 if self.profIndex == 0:
174 183 self.id_min = 0
175 184 self.id_max = nVoltProfiles
176 185
177 186 self.buffer[:, self.id_min:self.id_max,
178 187 :] = self.dataIn.data
179 188 self.profIndex += nVoltProfiles
180 189 self.id_min += nVoltProfiles
181 190 self.id_max += nVoltProfiles
182 191 else:
183 192 raise ValueError("The type object %s has %d profiles, it should just has %d profiles" % (
184 193 self.dataIn.type, self.dataIn.data.shape[1], nProfiles))
185 194 self.dataOut.flagNoData = True
186 195 return 0
187 196 else:
188 197 self.buffer[:, self.profIndex, :] = self.dataIn.data.copy()
189 198 self.profIndex += 1
190 199
191 200 if self.firstdatatime == None:
192 201 self.firstdatatime = self.dataIn.utctime
193 202
194 203 if self.profIndex == nProfiles:
195 204 self.__updateSpecFromVoltage()
196 205 self.__getFft()
197 206
198 207 self.dataOut.flagNoData = False
199 208 self.firstdatatime = None
200 209 self.profIndex = 0
201 210
202 211 return True
203 212
204 213 raise ValueError("The type of input object '%s' is not valid" % (
205 214 self.dataIn.type))
206 215
207 216 def __selectPairs(self, pairsList):
208 217
209 218 if not pairsList:
210 219 return
211 220
212 221 pairs = []
213 222 pairsIndex = []
214 223
215 224 for pair in pairsList:
216 225 if pair[0] not in self.dataOut.channelList or pair[1] not in self.dataOut.channelList:
217 226 continue
218 227 pairs.append(pair)
219 228 pairsIndex.append(pairs.index(pair))
220 229
221 230 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndex]
222 231 self.dataOut.pairsList = pairs
223 232
224 233 return
225 234
226 235 def __selectPairsByChannel(self, channelList=None):
227 236
228 237 if channelList == None:
229 238 return
230 239
231 240 pairsIndexListSelected = []
232 241 for pairIndex in self.dataOut.pairsIndexList:
233 242 # First pair
234 243 if self.dataOut.pairsList[pairIndex][0] not in channelList:
235 244 continue
236 245 # Second pair
237 246 if self.dataOut.pairsList[pairIndex][1] not in channelList:
238 247 continue
239 248
240 249 pairsIndexListSelected.append(pairIndex)
241 250
242 251 if not pairsIndexListSelected:
243 252 self.dataOut.data_cspc = None
244 253 self.dataOut.pairsList = []
245 254 return
246 255
247 256 self.dataOut.data_cspc = self.dataOut.data_cspc[pairsIndexListSelected]
248 257 self.dataOut.pairsList = [self.dataOut.pairsList[i]
249 258 for i in pairsIndexListSelected]
250 259
251 260 return
252 261
253 262 def selectChannels(self, channelList):
254 263
255 264 channelIndexList = []
256 265
257 266 for channel in channelList:
258 267 if channel not in self.dataOut.channelList:
259 268 raise ValueError("Error selecting channels, Channel %d is not valid.\nAvailable channels = %s" % (
260 269 channel, str(self.dataOut.channelList)))
261 270
262 271 index = self.dataOut.channelList.index(channel)
263 272 channelIndexList.append(index)
264 273
265 274 self.selectChannelsByIndex(channelIndexList)
266 275
267 276 def selectChannelsByIndex(self, channelIndexList):
268 277 """
269 278 Selecciona un bloque de datos en base a canales segun el channelIndexList
270 279
271 280 Input:
272 281 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
273 282
274 283 Affected:
275 284 self.dataOut.data_spc
276 285 self.dataOut.channelIndexList
277 286 self.dataOut.nChannels
278 287
279 288 Return:
280 289 None
281 290 """
282 291
283 292 for channelIndex in channelIndexList:
284 293 if channelIndex not in self.dataOut.channelIndexList:
285 294 raise ValueError("Error selecting channels: The value %d in channelIndexList is not valid.\nAvailable channel indexes = " % (
286 295 channelIndex, self.dataOut.channelIndexList))
287 296
288 297 # nChannels = len(channelIndexList)
289 298
290 299 data_spc = self.dataOut.data_spc[channelIndexList, :]
291 300 data_dc = self.dataOut.data_dc[channelIndexList, :]
292 301
293 302 self.dataOut.data_spc = data_spc
294 303 self.dataOut.data_dc = data_dc
295 304
296 305 self.dataOut.channelList = [
297 306 self.dataOut.channelList[i] for i in channelIndexList]
298 307 # self.dataOut.nChannels = nChannels
299 308
300 309 self.__selectPairsByChannel(self.dataOut.channelList)
301 310
302 311 return 1
303 312
304 313 def selectHeights(self, minHei, maxHei):
305 314 """
306 315 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
307 316 minHei <= height <= maxHei
308 317
309 318 Input:
310 319 minHei : valor minimo de altura a considerar
311 320 maxHei : valor maximo de altura a considerar
312 321
313 322 Affected:
314 323 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
315 324
316 325 Return:
317 326 1 si el metodo se ejecuto con exito caso contrario devuelve 0
318 327 """
319 328
320 329 if (minHei > maxHei):
321 330 raise ValueError("Error selecting heights: Height range (%d,%d) is not valid" % (
322 331 minHei, maxHei))
323 332
324 333 if (minHei < self.dataOut.heightList[0]):
325 334 minHei = self.dataOut.heightList[0]
326 335
327 336 if (maxHei > self.dataOut.heightList[-1]):
328 337 maxHei = self.dataOut.heightList[-1]
329 338
330 339 minIndex = 0
331 340 maxIndex = 0
332 341 heights = self.dataOut.heightList
333 342
334 343 inda = numpy.where(heights >= minHei)
335 344 indb = numpy.where(heights <= maxHei)
336 345
337 346 try:
338 347 minIndex = inda[0][0]
339 348 except:
340 349 minIndex = 0
341 350
342 351 try:
343 352 maxIndex = indb[0][-1]
344 353 except:
345 354 maxIndex = len(heights)
346 355
347 356 self.selectHeightsByIndex(minIndex, maxIndex)
348 357
349 358 return 1
350 359
351 360 def getBeaconSignal(self, tauindex=0, channelindex=0, hei_ref=None):
352 361 newheis = numpy.where(
353 362 self.dataOut.heightList > self.dataOut.radarControllerHeaderObj.Taus[tauindex])
354 363
355 364 if hei_ref != None:
356 365 newheis = numpy.where(self.dataOut.heightList > hei_ref)
357 366
358 367 minIndex = min(newheis[0])
359 368 maxIndex = max(newheis[0])
360 369 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
361 370 heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
362 371
363 372 # determina indices
364 373 nheis = int(self.dataOut.radarControllerHeaderObj.txB /
365 374 (self.dataOut.heightList[1] - self.dataOut.heightList[0]))
366 375 avg_dB = 10 * \
367 376 numpy.log10(numpy.sum(data_spc[channelindex, :, :], axis=0))
368 377 beacon_dB = numpy.sort(avg_dB)[-nheis:]
369 378 beacon_heiIndexList = []
370 379 for val in avg_dB.tolist():
371 380 if val >= beacon_dB[0]:
372 381 beacon_heiIndexList.append(avg_dB.tolist().index(val))
373 382
374 383 #data_spc = data_spc[:,:,beacon_heiIndexList]
375 384 data_cspc = None
376 385 if self.dataOut.data_cspc is not None:
377 386 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
378 387 #data_cspc = data_cspc[:,:,beacon_heiIndexList]
379 388
380 389 data_dc = None
381 390 if self.dataOut.data_dc is not None:
382 391 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
383 392 #data_dc = data_dc[:,beacon_heiIndexList]
384 393
385 394 self.dataOut.data_spc = data_spc
386 395 self.dataOut.data_cspc = data_cspc
387 396 self.dataOut.data_dc = data_dc
388 397 self.dataOut.heightList = heightList
389 398 self.dataOut.beacon_heiIndexList = beacon_heiIndexList
390 399
391 400 return 1
392 401
393 402 def selectHeightsByIndex(self, minIndex, maxIndex):
394 403 """
395 404 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
396 405 minIndex <= index <= maxIndex
397 406
398 407 Input:
399 408 minIndex : valor de indice minimo de altura a considerar
400 409 maxIndex : valor de indice maximo de altura a considerar
401 410
402 411 Affected:
403 412 self.dataOut.data_spc
404 413 self.dataOut.data_cspc
405 414 self.dataOut.data_dc
406 415 self.dataOut.heightList
407 416
408 417 Return:
409 418 1 si el metodo se ejecuto con exito caso contrario devuelve 0
410 419 """
411 420
412 421 if (minIndex < 0) or (minIndex > maxIndex):
413 422 raise ValueError("Error selecting heights: Index range (%d,%d) is not valid" % (
414 423 minIndex, maxIndex))
415 424
416 425 if (maxIndex >= self.dataOut.nHeights):
417 426 maxIndex = self.dataOut.nHeights - 1
418 427
419 428 # Spectra
420 429 data_spc = self.dataOut.data_spc[:, :, minIndex:maxIndex + 1]
421 430
422 431 data_cspc = None
423 432 if self.dataOut.data_cspc is not None:
424 433 data_cspc = self.dataOut.data_cspc[:, :, minIndex:maxIndex + 1]
425 434
426 435 data_dc = None
427 436 if self.dataOut.data_dc is not None:
428 437 data_dc = self.dataOut.data_dc[:, minIndex:maxIndex + 1]
429 438
430 439 self.dataOut.data_spc = data_spc
431 440 self.dataOut.data_cspc = data_cspc
432 441 self.dataOut.data_dc = data_dc
433 442
434 443 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex + 1]
435 444
436 445 return 1
437 446
438 447 def removeDC(self, mode=2):
439 448 jspectra = self.dataOut.data_spc
440 449 jcspectra = self.dataOut.data_cspc
441 450
442 451 num_chan = jspectra.shape[0]
443 452 num_hei = jspectra.shape[2]
444 453
445 454 if jcspectra is not None:
446 455 jcspectraExist = True
447 456 num_pairs = jcspectra.shape[0]
448 457 else:
449 458 jcspectraExist = False
450 459
451 460 freq_dc = int(jspectra.shape[1] / 2)
452 461 ind_vel = numpy.array([-2, -1, 1, 2]) + freq_dc
453 462 ind_vel = ind_vel.astype(int)
454 463
455 464 if ind_vel[0] < 0:
456 465 ind_vel[list(range(0, 1))] = ind_vel[list(range(0, 1))] + self.num_prof
457 466
458 467 if mode == 1:
459 468 jspectra[:, freq_dc, :] = (
460 469 jspectra[:, ind_vel[1], :] + jspectra[:, ind_vel[2], :]) / 2 # CORRECCION
461 470
462 471 if jcspectraExist:
463 472 jcspectra[:, freq_dc, :] = (
464 473 jcspectra[:, ind_vel[1], :] + jcspectra[:, ind_vel[2], :]) / 2
465 474
466 475 if mode == 2:
467 476
468 477 vel = numpy.array([-2, -1, 1, 2])
469 478 xx = numpy.zeros([4, 4])
470 479
471 480 for fil in range(4):
472 481 xx[fil, :] = vel[fil]**numpy.asarray(list(range(4)))
473 482
474 483 xx_inv = numpy.linalg.inv(xx)
475 484 xx_aux = xx_inv[0, :]
476 485
477 486 for ich in range(num_chan):
478 487 yy = jspectra[ich, ind_vel, :]
479 488 jspectra[ich, freq_dc, :] = numpy.dot(xx_aux, yy)
480 489
481 490 junkid = jspectra[ich, freq_dc, :] <= 0
482 491 cjunkid = sum(junkid)
483 492
484 493 if cjunkid.any():
485 494 jspectra[ich, freq_dc, junkid.nonzero()] = (
486 495 jspectra[ich, ind_vel[1], junkid] + jspectra[ich, ind_vel[2], junkid]) / 2
487 496
488 497 if jcspectraExist:
489 498 for ip in range(num_pairs):
490 499 yy = jcspectra[ip, ind_vel, :]
491 500 jcspectra[ip, freq_dc, :] = numpy.dot(xx_aux, yy)
492 501
493 502 self.dataOut.data_spc = jspectra
494 503 self.dataOut.data_cspc = jcspectra
495 504
496 505 return 1
497 506
498 507 def removeInterference(self, interf=2, hei_interf=None, nhei_interf=None, offhei_interf=None):
499 508
500 509 jspectra = self.dataOut.data_spc
501 510 jcspectra = self.dataOut.data_cspc
502 511 jnoise = self.dataOut.getNoise()
503 512 num_incoh = self.dataOut.nIncohInt
504 513
505 514 num_channel = jspectra.shape[0]
506 515 num_prof = jspectra.shape[1]
507 516 num_hei = jspectra.shape[2]
508 517
509 518 # hei_interf
510 519 if hei_interf is None:
511 520 count_hei = num_hei / 2 # Como es entero no importa
512 521 hei_interf = numpy.asmatrix(list(range(count_hei))) + num_hei - count_hei
513 522 hei_interf = numpy.asarray(hei_interf)[0]
514 523 # nhei_interf
515 524 if (nhei_interf == None):
516 525 nhei_interf = 5
517 526 if (nhei_interf < 1):
518 527 nhei_interf = 1
519 528 if (nhei_interf > count_hei):
520 529 nhei_interf = count_hei
521 530 if (offhei_interf == None):
522 531 offhei_interf = 0
523 532
524 533 ind_hei = list(range(num_hei))
525 534 # mask_prof = numpy.asarray(range(num_prof - 2)) + 1
526 535 # mask_prof[range(num_prof/2 - 1,len(mask_prof))] += 1
527 536 mask_prof = numpy.asarray(list(range(num_prof)))
528 537 num_mask_prof = mask_prof.size
529 538 comp_mask_prof = [0, num_prof / 2]
530 539
531 540 # noise_exist: Determina si la variable jnoise ha sido definida y contiene la informacion del ruido de cada canal
532 541 if (jnoise.size < num_channel or numpy.isnan(jnoise).any()):
533 542 jnoise = numpy.nan
534 543 noise_exist = jnoise[0] < numpy.Inf
535 544
536 545 # Subrutina de Remocion de la Interferencia
537 546 for ich in range(num_channel):
538 547 # Se ordena los espectros segun su potencia (menor a mayor)
539 548 power = jspectra[ich, mask_prof, :]
540 549 power = power[:, hei_interf]
541 550 power = power.sum(axis=0)
542 551 psort = power.ravel().argsort()
543 552
544 553 # Se estima la interferencia promedio en los Espectros de Potencia empleando
545 554 junkspc_interf = jspectra[ich, :, hei_interf[psort[list(range(
546 555 offhei_interf, nhei_interf + offhei_interf))]]]
547 556
548 557 if noise_exist:
549 558 # tmp_noise = jnoise[ich] / num_prof
550 559 tmp_noise = jnoise[ich]
551 560 junkspc_interf = junkspc_interf - tmp_noise
552 561 #junkspc_interf[:,comp_mask_prof] = 0
553 562
554 563 jspc_interf = junkspc_interf.sum(axis=0) / nhei_interf
555 564 jspc_interf = jspc_interf.transpose()
556 565 # Calculando el espectro de interferencia promedio
557 566 noiseid = numpy.where(
558 567 jspc_interf <= tmp_noise / numpy.sqrt(num_incoh))
559 568 noiseid = noiseid[0]
560 569 cnoiseid = noiseid.size
561 570 interfid = numpy.where(
562 571 jspc_interf > tmp_noise / numpy.sqrt(num_incoh))
563 572 interfid = interfid[0]
564 573 cinterfid = interfid.size
565 574
566 575 if (cnoiseid > 0):
567 576 jspc_interf[noiseid] = 0
568 577
569 578 # Expandiendo los perfiles a limpiar
570 579 if (cinterfid > 0):
571 580 new_interfid = (
572 581 numpy.r_[interfid - 1, interfid, interfid + 1] + num_prof) % num_prof
573 582 new_interfid = numpy.asarray(new_interfid)
574 583 new_interfid = {x for x in new_interfid}
575 584 new_interfid = numpy.array(list(new_interfid))
576 585 new_cinterfid = new_interfid.size
577 586 else:
578 587 new_cinterfid = 0
579 588
580 589 for ip in range(new_cinterfid):
581 590 ind = junkspc_interf[:, new_interfid[ip]].ravel().argsort()
582 591 jspc_interf[new_interfid[ip]
583 592 ] = junkspc_interf[ind[nhei_interf / 2], new_interfid[ip]]
584 593
585 594 jspectra[ich, :, ind_hei] = jspectra[ich, :,
586 595 ind_hei] - jspc_interf # Corregir indices
587 596
588 597 # Removiendo la interferencia del punto de mayor interferencia
589 598 ListAux = jspc_interf[mask_prof].tolist()
590 599 maxid = ListAux.index(max(ListAux))
591 600
592 601 if cinterfid > 0:
593 602 for ip in range(cinterfid * (interf == 2) - 1):
594 603 ind = (jspectra[ich, interfid[ip], :] < tmp_noise *
595 604 (1 + 1 / numpy.sqrt(num_incoh))).nonzero()
596 605 cind = len(ind)
597 606
598 607 if (cind > 0):
599 608 jspectra[ich, interfid[ip], ind] = tmp_noise * \
600 609 (1 + (numpy.random.uniform(cind) - 0.5) /
601 610 numpy.sqrt(num_incoh))
602 611
603 612 ind = numpy.array([-2, -1, 1, 2])
604 613 xx = numpy.zeros([4, 4])
605 614
606 615 for id1 in range(4):
607 616 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
608 617
609 618 xx_inv = numpy.linalg.inv(xx)
610 619 xx = xx_inv[:, 0]
611 620 ind = (ind + maxid + num_mask_prof) % num_mask_prof
612 621 yy = jspectra[ich, mask_prof[ind], :]
613 622 jspectra[ich, mask_prof[maxid], :] = numpy.dot(
614 623 yy.transpose(), xx)
615 624
616 625 indAux = (jspectra[ich, :, :] < tmp_noise *
617 626 (1 - 1 / numpy.sqrt(num_incoh))).nonzero()
618 627 jspectra[ich, indAux[0], indAux[1]] = tmp_noise * \
619 628 (1 - 1 / numpy.sqrt(num_incoh))
620 629
621 630 # Remocion de Interferencia en el Cross Spectra
622 631 if jcspectra is None:
623 632 return jspectra, jcspectra
624 633 num_pairs = jcspectra.size / (num_prof * num_hei)
625 634 jcspectra = jcspectra.reshape(num_pairs, num_prof, num_hei)
626 635
627 636 for ip in range(num_pairs):
628 637
629 638 #-------------------------------------------
630 639
631 640 cspower = numpy.abs(jcspectra[ip, mask_prof, :])
632 641 cspower = cspower[:, hei_interf]
633 642 cspower = cspower.sum(axis=0)
634 643
635 644 cspsort = cspower.ravel().argsort()
636 645 junkcspc_interf = jcspectra[ip, :, hei_interf[cspsort[list(range(
637 646 offhei_interf, nhei_interf + offhei_interf))]]]
638 647 junkcspc_interf = junkcspc_interf.transpose()
639 648 jcspc_interf = junkcspc_interf.sum(axis=1) / nhei_interf
640 649
641 650 ind = numpy.abs(jcspc_interf[mask_prof]).ravel().argsort()
642 651
643 652 median_real = numpy.median(numpy.real(
644 653 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof / 4))]], :]))
645 654 median_imag = numpy.median(numpy.imag(
646 655 junkcspc_interf[mask_prof[ind[list(range(3 * num_prof / 4))]], :]))
647 656 junkcspc_interf[comp_mask_prof, :] = numpy.complex(
648 657 median_real, median_imag)
649 658
650 659 for iprof in range(num_prof):
651 660 ind = numpy.abs(junkcspc_interf[iprof, :]).ravel().argsort()
652 661 jcspc_interf[iprof] = junkcspc_interf[iprof,
653 662 ind[nhei_interf / 2]]
654 663
655 664 # Removiendo la Interferencia
656 665 jcspectra[ip, :, ind_hei] = jcspectra[ip,
657 666 :, ind_hei] - jcspc_interf
658 667
659 668 ListAux = numpy.abs(jcspc_interf[mask_prof]).tolist()
660 669 maxid = ListAux.index(max(ListAux))
661 670
662 671 ind = numpy.array([-2, -1, 1, 2])
663 672 xx = numpy.zeros([4, 4])
664 673
665 674 for id1 in range(4):
666 675 xx[:, id1] = ind[id1]**numpy.asarray(list(range(4)))
667 676
668 677 xx_inv = numpy.linalg.inv(xx)
669 678 xx = xx_inv[:, 0]
670 679
671 680 ind = (ind + maxid + num_mask_prof) % num_mask_prof
672 681 yy = jcspectra[ip, mask_prof[ind], :]
673 682 jcspectra[ip, mask_prof[maxid], :] = numpy.dot(yy.transpose(), xx)
674 683
675 684 # Guardar Resultados
676 685 self.dataOut.data_spc = jspectra
677 686 self.dataOut.data_cspc = jcspectra
678 687
679 688 return 1
680 689
681 690 def setRadarFrequency(self, frequency=None):
682 691
683 692 if frequency != None:
684 693 self.dataOut.frequency = frequency
685 694
686 695 return 1
687 696
688 697 def getNoise(self, minHei=None, maxHei=None, minVel=None, maxVel=None):
689 698 # validacion de rango
690 699 if minHei == None:
691 700 minHei = self.dataOut.heightList[0]
692 701
693 702 if maxHei == None:
694 703 maxHei = self.dataOut.heightList[-1]
695 704
696 705 if (minHei < self.dataOut.heightList[0]) or (minHei > maxHei):
697 706 print('minHei: %.2f is out of the heights range' % (minHei))
698 707 print('minHei is setting to %.2f' % (self.dataOut.heightList[0]))
699 708 minHei = self.dataOut.heightList[0]
700 709
701 710 if (maxHei > self.dataOut.heightList[-1]) or (maxHei < minHei):
702 711 print('maxHei: %.2f is out of the heights range' % (maxHei))
703 712 print('maxHei is setting to %.2f' % (self.dataOut.heightList[-1]))
704 713 maxHei = self.dataOut.heightList[-1]
705 714
706 715 # validacion de velocidades
707 716 velrange = self.dataOut.getVelRange(1)
708 717
709 718 if minVel == None:
710 719 minVel = velrange[0]
711 720
712 721 if maxVel == None:
713 722 maxVel = velrange[-1]
714 723
715 724 if (minVel < velrange[0]) or (minVel > maxVel):
716 725 print('minVel: %.2f is out of the velocity range' % (minVel))
717 726 print('minVel is setting to %.2f' % (velrange[0]))
718 727 minVel = velrange[0]
719 728
720 729 if (maxVel > velrange[-1]) or (maxVel < minVel):
721 730 print('maxVel: %.2f is out of the velocity range' % (maxVel))
722 731 print('maxVel is setting to %.2f' % (velrange[-1]))
723 732 maxVel = velrange[-1]
724 733
725 734 # seleccion de indices para rango
726 735 minIndex = 0
727 736 maxIndex = 0
728 737 heights = self.dataOut.heightList
729 738
730 739 inda = numpy.where(heights >= minHei)
731 740 indb = numpy.where(heights <= maxHei)
732 741
733 742 try:
734 743 minIndex = inda[0][0]
735 744 except:
736 745 minIndex = 0
737 746
738 747 try:
739 748 maxIndex = indb[0][-1]
740 749 except:
741 750 maxIndex = len(heights)
742 751
743 752 if (minIndex < 0) or (minIndex > maxIndex):
744 753 raise ValueError("some value in (%d,%d) is not valid" % (
745 754 minIndex, maxIndex))
746 755
747 756 if (maxIndex >= self.dataOut.nHeights):
748 757 maxIndex = self.dataOut.nHeights - 1
749 758
750 759 # seleccion de indices para velocidades
751 760 indminvel = numpy.where(velrange >= minVel)
752 761 indmaxvel = numpy.where(velrange <= maxVel)
753 762 try:
754 763 minIndexVel = indminvel[0][0]
755 764 except:
756 765 minIndexVel = 0
757 766
758 767 try:
759 768 maxIndexVel = indmaxvel[0][-1]
760 769 except:
761 770 maxIndexVel = len(velrange)
762 771
763 772 # seleccion del espectro
764 773 data_spc = self.dataOut.data_spc[:,
765 774 minIndexVel:maxIndexVel + 1, minIndex:maxIndex + 1]
766 775 # estimacion de ruido
767 776 noise = numpy.zeros(self.dataOut.nChannels)
768 777
769 778 for channel in range(self.dataOut.nChannels):
770 779 daux = data_spc[channel, :, :]
771 780 noise[channel] = hildebrand_sekhon(daux, self.dataOut.nIncohInt)
772 781
773 782 self.dataOut.noise_estimation = noise.copy()
774 783
775 784 return 1
776 785
777
786 @MPDecorator
778 787 class IncohInt(Operation):
779 788
780 789 __profIndex = 0
781 790 __withOverapping = False
782 791
783 792 __byTime = False
784 793 __initime = None
785 794 __lastdatatime = None
786 795 __integrationtime = None
787 796
788 797 __buffer_spc = None
789 798 __buffer_cspc = None
790 799 __buffer_dc = None
791 800
792 801 __dataReady = False
793 802
794 803 __timeInterval = None
795 804
796 805 n = None
797 806
798 def __init__(self, **kwargs):
807 def __init__(self):#, **kwargs):
808
809 Operation.__init__(self)#, **kwargs)
810
799 811
800 Operation.__init__(self, **kwargs)
801 812 # self.isConfig = False
802 813
803 814 def setup(self, n=None, timeInterval=None, overlapping=False):
804 815 """
805 816 Set the parameters of the integration class.
806 817
807 818 Inputs:
808 819
809 820 n : Number of coherent integrations
810 821 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
811 822 overlapping :
812 823
813 824 """
814 825
815 826 self.__initime = None
816 827 self.__lastdatatime = 0
817 828
818 829 self.__buffer_spc = 0
819 830 self.__buffer_cspc = 0
820 831 self.__buffer_dc = 0
821 832
822 833 self.__profIndex = 0
823 834 self.__dataReady = False
824 835 self.__byTime = False
825 836
826 837 if n is None and timeInterval is None:
827 838 raise ValueError("n or timeInterval should be specified ...")
828 839
829 840 if n is not None:
830 841 self.n = int(n)
831 842 else:
832 843 # if (type(timeInterval)!=integer) -> change this line
833 844 self.__integrationtime = int(timeInterval)
834 845 self.n = None
835 846 self.__byTime = True
836 847
837 848 def putData(self, data_spc, data_cspc, data_dc):
838 849 """
839 850 Add a profile to the __buffer_spc and increase in one the __profileIndex
840 851
841 852 """
842 853
843 854 self.__buffer_spc += data_spc
844 855
845 856 if data_cspc is None:
846 857 self.__buffer_cspc = None
847 858 else:
848 859 self.__buffer_cspc += data_cspc
849 860
850 861 if data_dc is None:
851 862 self.__buffer_dc = None
852 863 else:
853 864 self.__buffer_dc += data_dc
854 865
855 866 self.__profIndex += 1
856 867
857 868 return
858 869
859 870 def pushData(self):
860 871 """
861 872 Return the sum of the last profiles and the profiles used in the sum.
862 873
863 874 Affected:
864 875
865 876 self.__profileIndex
866 877
867 878 """
868 879
869 880 data_spc = self.__buffer_spc
870 881 data_cspc = self.__buffer_cspc
871 882 data_dc = self.__buffer_dc
872 883 n = self.__profIndex
873 884
874 885 self.__buffer_spc = 0
875 886 self.__buffer_cspc = 0
876 887 self.__buffer_dc = 0
877 888 self.__profIndex = 0
878 889
879 890 return data_spc, data_cspc, data_dc, n
880 891
881 892 def byProfiles(self, *args):
882 893
883 894 self.__dataReady = False
884 895 avgdata_spc = None
885 896 avgdata_cspc = None
886 897 avgdata_dc = None
887 898
888 899 self.putData(*args)
889 900
890 901 if self.__profIndex == self.n:
891 902
892 903 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
893 904 self.n = n
894 905 self.__dataReady = True
895 906
896 907 return avgdata_spc, avgdata_cspc, avgdata_dc
897 908
898 909 def byTime(self, datatime, *args):
899 910
900 911 self.__dataReady = False
901 912 avgdata_spc = None
902 913 avgdata_cspc = None
903 914 avgdata_dc = None
904 915
905 916 self.putData(*args)
906 917
907 918 if (datatime - self.__initime) >= self.__integrationtime:
908 919 avgdata_spc, avgdata_cspc, avgdata_dc, n = self.pushData()
909 920 self.n = n
910 921 self.__dataReady = True
911 922
912 923 return avgdata_spc, avgdata_cspc, avgdata_dc
913 924
914 925 def integrate(self, datatime, *args):
915 926
916 927 if self.__profIndex == 0:
917 928 self.__initime = datatime
918 929
919 930 if self.__byTime:
920 931 avgdata_spc, avgdata_cspc, avgdata_dc = self.byTime(
921 932 datatime, *args)
922 933 else:
923 934 avgdata_spc, avgdata_cspc, avgdata_dc = self.byProfiles(*args)
924 935
925 936 if not self.__dataReady:
926 937 return None, None, None, None
927 938
928 939 return self.__initime, avgdata_spc, avgdata_cspc, avgdata_dc
929 940
930 941 def run(self, dataOut, n=None, timeInterval=None, overlapping=False):
931 942 if n == 1:
932 943 return
933
944
934 945 dataOut.flagNoData = True
935 946
936 947 if not self.isConfig:
937 948 self.setup(n, timeInterval, overlapping)
938 949 self.isConfig = True
939 950
940 951 avgdatatime, avgdata_spc, avgdata_cspc, avgdata_dc = self.integrate(dataOut.utctime,
941 952 dataOut.data_spc,
942 953 dataOut.data_cspc,
943 954 dataOut.data_dc)
944 955
945 956 if self.__dataReady:
946 957
947 958 dataOut.data_spc = avgdata_spc
948 959 dataOut.data_cspc = avgdata_cspc
949 960 dataOut.data_dc = avgdata_dc
950 961
951 962 dataOut.nIncohInt *= self.n
952 963 dataOut.utctime = avgdatatime
953 dataOut.flagNoData = False No newline at end of file
964 dataOut.flagNoData = False
965
966 return dataOut No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,1321 +1,1335
1 1 import sys
2 2 import numpy
3 3 from scipy import interpolate
4 4 #TODO
5 5 #from schainpy import cSchain
6 from .jroproc_base import ProcessingUnit, Operation
6 from schainpy.model.proc.jroproc_base import ProcessingUnit, MPDecorator, Operation
7 7 from schainpy.model.data.jrodata import Voltage
8 from time import time
9 8 from schainpy.utils import log
9 from time import time
10 10
11 11
12 @MPDecorator
12 13 class VoltageProc(ProcessingUnit):
14
15 METHODS = {} #yong
13 16
17 def __init__(self):#, **kwargs): #yong
14 18
15 def __init__(self, **kwargs):
16
17 ProcessingUnit.__init__(self, **kwargs)
19 ProcessingUnit.__init__(self)#, **kwargs)
18 20
19 21 # self.objectDict = {}
20 22 self.dataOut = Voltage()
21 23 self.flip = 1
24 self.setupReq = False #yong
22 25
23 26 def run(self):
27
24 28 if self.dataIn.type == 'AMISR':
25 29 self.__updateObjFromAmisrInput()
26 30
27 31 if self.dataIn.type == 'Voltage':
28 32 self.dataOut.copy(self.dataIn)
29 33
30 34 # self.dataOut.copy(self.dataIn)
31 35
32 36 def __updateObjFromAmisrInput(self):
33 37
34 38 self.dataOut.timeZone = self.dataIn.timeZone
35 39 self.dataOut.dstFlag = self.dataIn.dstFlag
36 40 self.dataOut.errorCount = self.dataIn.errorCount
37 41 self.dataOut.useLocalTime = self.dataIn.useLocalTime
38 42
39 43 self.dataOut.flagNoData = self.dataIn.flagNoData
40 44 self.dataOut.data = self.dataIn.data
41 45 self.dataOut.utctime = self.dataIn.utctime
42 46 self.dataOut.channelList = self.dataIn.channelList
43 47 # self.dataOut.timeInterval = self.dataIn.timeInterval
44 48 self.dataOut.heightList = self.dataIn.heightList
45 49 self.dataOut.nProfiles = self.dataIn.nProfiles
46 50
47 51 self.dataOut.nCohInt = self.dataIn.nCohInt
48 52 self.dataOut.ippSeconds = self.dataIn.ippSeconds
49 53 self.dataOut.frequency = self.dataIn.frequency
50 54
51 55 self.dataOut.azimuth = self.dataIn.azimuth
52 56 self.dataOut.zenith = self.dataIn.zenith
53 57
54 58 self.dataOut.beam.codeList = self.dataIn.beam.codeList
55 59 self.dataOut.beam.azimuthList = self.dataIn.beam.azimuthList
56 60 self.dataOut.beam.zenithList = self.dataIn.beam.zenithList
57 61 #
58 62 # pass#
59 63 #
60 64 # def init(self):
61 65 #
62 66 #
63 67 # if self.dataIn.type == 'AMISR':
64 68 # self.__updateObjFromAmisrInput()
65 69 #
66 70 # if self.dataIn.type == 'Voltage':
67 71 # self.dataOut.copy(self.dataIn)
68 72 # # No necesita copiar en cada init() los atributos de dataIn
69 73 # # la copia deberia hacerse por cada nuevo bloque de datos
70 74
71 75 def selectChannels(self, channelList):
72 76
73 77 channelIndexList = []
74 78
75 79 for channel in channelList:
76 80 if channel not in self.dataOut.channelList:
77 81 raise ValueError("Channel %d is not in %s" %(channel, str(self.dataOut.channelList)))
78 82
79 83 index = self.dataOut.channelList.index(channel)
80 84 channelIndexList.append(index)
81 85
82 86 self.selectChannelsByIndex(channelIndexList)
83 87
84 88 def selectChannelsByIndex(self, channelIndexList):
85 89 """
86 90 Selecciona un bloque de datos en base a canales segun el channelIndexList
87 91
88 92 Input:
89 93 channelIndexList : lista sencilla de canales a seleccionar por ej. [2,3,7]
90 94
91 95 Affected:
92 96 self.dataOut.data
93 97 self.dataOut.channelIndexList
94 98 self.dataOut.nChannels
95 99 self.dataOut.m_ProcessingHeader.totalSpectra
96 100 self.dataOut.systemHeaderObj.numChannels
97 101 self.dataOut.m_ProcessingHeader.blockSize
98 102
99 103 Return:
100 104 None
101 105 """
102 106
103 107 for channelIndex in channelIndexList:
104 108 if channelIndex not in self.dataOut.channelIndexList:
105 109 print(channelIndexList)
106 110 raise ValueError("The value %d in channelIndexList is not valid" %channelIndex)
107 111
108 112 if self.dataOut.flagDataAsBlock:
109 113 """
110 114 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
111 115 """
112 116 data = self.dataOut.data[channelIndexList,:,:]
113 117 else:
114 118 data = self.dataOut.data[channelIndexList,:]
115 119
116 120 self.dataOut.data = data
117 121 self.dataOut.channelList = [self.dataOut.channelList[i] for i in channelIndexList]
118 122 # self.dataOut.nChannels = nChannels
119 123
120 124 return 1
121 125
122 126 def selectHeights(self, minHei=None, maxHei=None):
123 127 """
124 128 Selecciona un bloque de datos en base a un grupo de valores de alturas segun el rango
125 129 minHei <= height <= maxHei
126 130
127 131 Input:
128 132 minHei : valor minimo de altura a considerar
129 133 maxHei : valor maximo de altura a considerar
130 134
131 135 Affected:
132 136 Indirectamente son cambiados varios valores a travez del metodo selectHeightsByIndex
133 137
134 138 Return:
135 139 1 si el metodo se ejecuto con exito caso contrario devuelve 0
136 140 """
137 141
138 142 if minHei == None:
139 143 minHei = self.dataOut.heightList[0]
140 144
141 145 if maxHei == None:
142 146 maxHei = self.dataOut.heightList[-1]
143 147
144 148 if (minHei < self.dataOut.heightList[0]):
145 149 minHei = self.dataOut.heightList[0]
146 150
147 151 if (maxHei > self.dataOut.heightList[-1]):
148 152 maxHei = self.dataOut.heightList[-1]
149 153
150 154 minIndex = 0
151 155 maxIndex = 0
152 156 heights = self.dataOut.heightList
153 157
154 158 inda = numpy.where(heights >= minHei)
155 159 indb = numpy.where(heights <= maxHei)
156 160
157 161 try:
158 162 minIndex = inda[0][0]
159 163 except:
160 164 minIndex = 0
161 165
162 166 try:
163 167 maxIndex = indb[0][-1]
164 168 except:
165 169 maxIndex = len(heights)
166 170
167 171 self.selectHeightsByIndex(minIndex, maxIndex)
168 172
169 173 return 1
170 174
171 175
172 176 def selectHeightsByIndex(self, minIndex, maxIndex):
173 177 """
174 178 Selecciona un bloque de datos en base a un grupo indices de alturas segun el rango
175 179 minIndex <= index <= maxIndex
176 180
177 181 Input:
178 182 minIndex : valor de indice minimo de altura a considerar
179 183 maxIndex : valor de indice maximo de altura a considerar
180 184
181 185 Affected:
182 186 self.dataOut.data
183 187 self.dataOut.heightList
184 188
185 189 Return:
186 190 1 si el metodo se ejecuto con exito caso contrario devuelve 0
187 191 """
188 192
189 193 if (minIndex < 0) or (minIndex > maxIndex):
190 194 raise ValueError("Height index range (%d,%d) is not valid" % (minIndex, maxIndex))
191 195
192 196 if (maxIndex >= self.dataOut.nHeights):
193 197 maxIndex = self.dataOut.nHeights
194 198
195 199 #voltage
196 200 if self.dataOut.flagDataAsBlock:
197 201 """
198 202 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
199 203 """
200 204 data = self.dataOut.data[:,:, minIndex:maxIndex]
201 205 else:
202 206 data = self.dataOut.data[:, minIndex:maxIndex]
203 207
204 208 # firstHeight = self.dataOut.heightList[minIndex]
205 209
206 210 self.dataOut.data = data
207 211 self.dataOut.heightList = self.dataOut.heightList[minIndex:maxIndex]
208 212
209 213 if self.dataOut.nHeights <= 1:
210 214 raise ValueError("selectHeights: Too few heights. Current number of heights is %d" %(self.dataOut.nHeights))
211 215
212 216 return 1
213 217
214 218
215 219 def filterByHeights(self, window):
216 220
217 221 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
218 222
219 223 if window == None:
220 224 window = (self.dataOut.radarControllerHeaderObj.txA/self.dataOut.radarControllerHeaderObj.nBaud) / deltaHeight
221 225
222 226 newdelta = deltaHeight * window
223 227 r = self.dataOut.nHeights % window
224 228 newheights = (self.dataOut.nHeights-r)/window
225 229
226 230 if newheights <= 1:
227 231 raise ValueError("filterByHeights: Too few heights. Current number of heights is %d and window is %d" %(self.dataOut.nHeights, window))
228 232
229 233 if self.dataOut.flagDataAsBlock:
230 234 """
231 235 Si la data es obtenida por bloques, dimension = [nChannels, nProfiles, nHeis]
232 236 """
233 237 buffer = self.dataOut.data[:, :, 0:self.dataOut.nHeights-r]
234 238 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nProfiles,self.dataOut.nHeights/window,window)
235 239 buffer = numpy.sum(buffer,3)
236 240
237 241 else:
238 242 buffer = self.dataOut.data[:,0:self.dataOut.nHeights-r]
239 243 buffer = buffer.reshape(self.dataOut.nChannels,self.dataOut.nHeights/window,window)
240 244 buffer = numpy.sum(buffer,2)
241 245
242 246 self.dataOut.data = buffer
243 247 self.dataOut.heightList = self.dataOut.heightList[0] + numpy.arange( newheights )*newdelta
244 248 self.dataOut.windowOfFilter = window
245 249
246 250 def setH0(self, h0, deltaHeight = None):
247 251
248 252 if not deltaHeight:
249 253 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
250 254
251 255 nHeights = self.dataOut.nHeights
252 256
253 257 newHeiRange = h0 + numpy.arange(nHeights)*deltaHeight
254 258
255 259 self.dataOut.heightList = newHeiRange
256 260
257 261 def deFlip(self, channelList = []):
258 262
259 263 data = self.dataOut.data.copy()
260 264
261 265 if self.dataOut.flagDataAsBlock:
262 266 flip = self.flip
263 267 profileList = list(range(self.dataOut.nProfiles))
264 268
265 269 if not channelList:
266 270 for thisProfile in profileList:
267 271 data[:,thisProfile,:] = data[:,thisProfile,:]*flip
268 272 flip *= -1.0
269 273 else:
270 274 for thisChannel in channelList:
271 275 if thisChannel not in self.dataOut.channelList:
272 276 continue
273 277
274 278 for thisProfile in profileList:
275 279 data[thisChannel,thisProfile,:] = data[thisChannel,thisProfile,:]*flip
276 280 flip *= -1.0
277 281
278 282 self.flip = flip
279 283
280 284 else:
281 285 if not channelList:
282 286 data[:,:] = data[:,:]*self.flip
283 287 else:
284 288 for thisChannel in channelList:
285 289 if thisChannel not in self.dataOut.channelList:
286 290 continue
287 291
288 292 data[thisChannel,:] = data[thisChannel,:]*self.flip
289 293
290 294 self.flip *= -1.
291 295
292 296 self.dataOut.data = data
293 297
294 298 def setRadarFrequency(self, frequency=None):
295 299
296 300 if frequency != None:
297 301 self.dataOut.frequency = frequency
298 302
299 303 return 1
300 304
301 305 def interpolateHeights(self, topLim, botLim):
302 306 #69 al 72 para julia
303 307 #82-84 para meteoros
304 308 if len(numpy.shape(self.dataOut.data))==2:
305 309 sampInterp = (self.dataOut.data[:,botLim-1] + self.dataOut.data[:,topLim+1])/2
306 310 sampInterp = numpy.transpose(numpy.tile(sampInterp,(topLim-botLim + 1,1)))
307 311 #self.dataOut.data[:,botLim:limSup+1] = sampInterp
308 312 self.dataOut.data[:,botLim:topLim+1] = sampInterp
309 313 else:
310 314 nHeights = self.dataOut.data.shape[2]
311 315 x = numpy.hstack((numpy.arange(botLim),numpy.arange(topLim+1,nHeights)))
312 316 y = self.dataOut.data[:,:,list(range(botLim))+list(range(topLim+1,nHeights))]
313 317 f = interpolate.interp1d(x, y, axis = 2)
314 318 xnew = numpy.arange(botLim,topLim+1)
315 319 ynew = f(xnew)
316 320
317 321 self.dataOut.data[:,:,botLim:topLim+1] = ynew
318 322
319 323 # import collections
320
324 @MPDecorator
321 325 class CohInt(Operation):
322 326
323 327 isConfig = False
324 328 __profIndex = 0
325 329 __byTime = False
326 330 __initime = None
327 331 __lastdatatime = None
328 332 __integrationtime = None
329 333 __buffer = None
330 334 __bufferStride = []
331 335 __dataReady = False
332 336 __profIndexStride = 0
333 337 __dataToPutStride = False
334 338 n = None
335 339
336 def __init__(self, **kwargs):
340 def __init__(self):#, **kwargs):
337 341
338 Operation.__init__(self, **kwargs)
342 Operation.__init__(self)#, **kwargs)
339 343
340 344 # self.isConfig = False
341 345
342 346 def setup(self, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False):
343 347 """
344 348 Set the parameters of the integration class.
345 349
346 350 Inputs:
347 351
348 352 n : Number of coherent integrations
349 353 timeInterval : Time of integration. If the parameter "n" is selected this one does not work
350 354 overlapping :
351 355 """
352 356
353 357 self.__initime = None
354 358 self.__lastdatatime = 0
355 359 self.__buffer = None
356 360 self.__dataReady = False
357 361 self.byblock = byblock
358 362 self.stride = stride
359 363
360 364 if n == None and timeInterval == None:
361 365 raise ValueError("n or timeInterval should be specified ...")
362 366
363 367 if n != None:
364 368 self.n = n
365 369 self.__byTime = False
366 370 else:
367 371 self.__integrationtime = timeInterval #* 60. #if (type(timeInterval)!=integer) -> change this line
368 372 self.n = 9999
369 373 self.__byTime = True
370 374
371 375 if overlapping:
372 376 self.__withOverlapping = True
373 377 self.__buffer = None
374 378 else:
375 379 self.__withOverlapping = False
376 380 self.__buffer = 0
377 381
378 382 self.__profIndex = 0
379 383
380 384 def putData(self, data):
381 385
382 386 """
383 387 Add a profile to the __buffer and increase in one the __profileIndex
384 388
385 389 """
386 390
387 391 if not self.__withOverlapping:
388 392 self.__buffer += data.copy()
389 393 self.__profIndex += 1
390 394 return
391 395
392 396 #Overlapping data
393 397 nChannels, nHeis = data.shape
394 398 data = numpy.reshape(data, (1, nChannels, nHeis))
395 399
396 400 #If the buffer is empty then it takes the data value
397 401 if self.__buffer is None:
398 402 self.__buffer = data
399 403 self.__profIndex += 1
400 404 return
401 405
402 406 #If the buffer length is lower than n then stakcing the data value
403 407 if self.__profIndex < self.n:
404 408 self.__buffer = numpy.vstack((self.__buffer, data))
405 409 self.__profIndex += 1
406 410 return
407 411
408 412 #If the buffer length is equal to n then replacing the last buffer value with the data value
409 413 self.__buffer = numpy.roll(self.__buffer, -1, axis=0)
410 414 self.__buffer[self.n-1] = data
411 415 self.__profIndex = self.n
412 416 return
413 417
414 418
415 419 def pushData(self):
416 420 """
417 421 Return the sum of the last profiles and the profiles used in the sum.
418 422
419 423 Affected:
420 424
421 425 self.__profileIndex
422 426
423 427 """
424 428
425 429 if not self.__withOverlapping:
426 430 data = self.__buffer
427 431 n = self.__profIndex
428 432
429 433 self.__buffer = 0
430 434 self.__profIndex = 0
431 435
432 436 return data, n
433 437
434 438 #Integration with Overlapping
435 439 data = numpy.sum(self.__buffer, axis=0)
436 440 # print data
437 441 # raise
438 442 n = self.__profIndex
439 443
440 444 return data, n
441 445
442 446 def byProfiles(self, data):
443 447
444 448 self.__dataReady = False
445 449 avgdata = None
446 450 # n = None
447 451 # print data
448 452 # raise
449 453 self.putData(data)
450 454
451 455 if self.__profIndex == self.n:
452 456 avgdata, n = self.pushData()
453 457 self.__dataReady = True
454 458
455 459 return avgdata
456 460
457 461 def byTime(self, data, datatime):
458 462
459 463 self.__dataReady = False
460 464 avgdata = None
461 465 n = None
462 466
463 467 self.putData(data)
464 468
465 469 if (datatime - self.__initime) >= self.__integrationtime:
466 470 avgdata, n = self.pushData()
467 471 self.n = n
468 472 self.__dataReady = True
469 473
470 474 return avgdata
471 475
472 476 def integrateByStride(self, data, datatime):
473 477 # print data
474 478 if self.__profIndex == 0:
475 479 self.__buffer = [[data.copy(), datatime]]
476 480 else:
477 481 self.__buffer.append([data.copy(),datatime])
478 482 self.__profIndex += 1
479 483 self.__dataReady = False
480 484
481 485 if self.__profIndex == self.n * self.stride :
482 486 self.__dataToPutStride = True
483 487 self.__profIndexStride = 0
484 488 self.__profIndex = 0
485 489 self.__bufferStride = []
486 490 for i in range(self.stride):
487 491 current = self.__buffer[i::self.stride]
488 492 data = numpy.sum([t[0] for t in current], axis=0)
489 493 avgdatatime = numpy.average([t[1] for t in current])
490 494 # print data
491 495 self.__bufferStride.append((data, avgdatatime))
492 496
493 497 if self.__dataToPutStride:
494 498 self.__dataReady = True
495 499 self.__profIndexStride += 1
496 500 if self.__profIndexStride == self.stride:
497 501 self.__dataToPutStride = False
498 502 # print self.__bufferStride[self.__profIndexStride - 1]
499 503 # raise
500 504 return self.__bufferStride[self.__profIndexStride - 1]
501 505
502 506
503 507 return None, None
504 508
505 509 def integrate(self, data, datatime=None):
506 510
507 511 if self.__initime == None:
508 512 self.__initime = datatime
509 513
510 514 if self.__byTime:
511 515 avgdata = self.byTime(data, datatime)
512 516 else:
513 517 avgdata = self.byProfiles(data)
514 518
515 519
516 520 self.__lastdatatime = datatime
517 521
518 522 if avgdata is None:
519 523 return None, None
520 524
521 525 avgdatatime = self.__initime
522 526
523 527 deltatime = datatime - self.__lastdatatime
524 528
525 529 if not self.__withOverlapping:
526 530 self.__initime = datatime
527 531 else:
528 532 self.__initime += deltatime
529 533
530 534 return avgdata, avgdatatime
531 535
532 536 def integrateByBlock(self, dataOut):
533 537
534 538 times = int(dataOut.data.shape[1]/self.n)
535 539 avgdata = numpy.zeros((dataOut.nChannels, times, dataOut.nHeights), dtype=numpy.complex)
536 540
537 541 id_min = 0
538 542 id_max = self.n
539 543
540 544 for i in range(times):
541 545 junk = dataOut.data[:,id_min:id_max,:]
542 546 avgdata[:,i,:] = junk.sum(axis=1)
543 547 id_min += self.n
544 548 id_max += self.n
545 549
546 550 timeInterval = dataOut.ippSeconds*self.n
547 551 avgdatatime = (times - 1) * timeInterval + dataOut.utctime
548 552 self.__dataReady = True
549 553 return avgdata, avgdatatime
550 554
551 555 def run(self, dataOut, n=None, timeInterval=None, stride=None, overlapping=False, byblock=False, **kwargs):
556
552 557 if not self.isConfig:
553 558 self.setup(n=n, stride=stride, timeInterval=timeInterval, overlapping=overlapping, byblock=byblock, **kwargs)
554 559 self.isConfig = True
555 560
556 561 if dataOut.flagDataAsBlock:
557 562 """
558 563 Si la data es leida por bloques, dimension = [nChannels, nProfiles, nHeis]
559 564 """
560 565 avgdata, avgdatatime = self.integrateByBlock(dataOut)
561 566 dataOut.nProfiles /= self.n
562 567 else:
563 568 if stride is None:
564 569 avgdata, avgdatatime = self.integrate(dataOut.data, dataOut.utctime)
565 570 else:
566 571 avgdata, avgdatatime = self.integrateByStride(dataOut.data, dataOut.utctime)
567 572
568 573
569 574 # dataOut.timeInterval *= n
570 575 dataOut.flagNoData = True
571 576
572 577 if self.__dataReady:
573 578 dataOut.data = avgdata
574 579 dataOut.nCohInt *= self.n
575 580 dataOut.utctime = avgdatatime
576 581 # print avgdata, avgdatatime
577 582 # raise
578 583 # dataOut.timeInterval = dataOut.ippSeconds * dataOut.nCohInt
579 584 dataOut.flagNoData = False
580
585 return dataOut
586 @MPDecorator
581 587 class Decoder(Operation):
582 588
583 589 isConfig = False
584 590 __profIndex = 0
585 591
586 592 code = None
587 593
588 594 nCode = None
589 595 nBaud = None
590 596
591 def __init__(self, **kwargs):
597 def __init__(self):#, **kwargs):
592 598
593 Operation.__init__(self, **kwargs)
599 Operation.__init__(self)#, **kwargs)
594 600
595 601 self.times = None
596 602 self.osamp = None
597 603 # self.__setValues = False
598 self.isConfig = False
599
604 # self.isConfig = False
605 self.setupReq = False
600 606 def setup(self, code, osamp, dataOut):
601 607
602 608 self.__profIndex = 0
603 609
604 610 self.code = code
605 611
606 612 self.nCode = len(code)
607 613 self.nBaud = len(code[0])
608 614
609 615 if (osamp != None) and (osamp >1):
610 616 self.osamp = osamp
611 617 self.code = numpy.repeat(code, repeats=self.osamp, axis=1)
612 618 self.nBaud = self.nBaud*self.osamp
613 619
614 620 self.__nChannels = dataOut.nChannels
615 621 self.__nProfiles = dataOut.nProfiles
616 622 self.__nHeis = dataOut.nHeights
617 623
618 624 if self.__nHeis < self.nBaud:
619 625 raise ValueError('Number of heights (%d) should be greater than number of bauds (%d)' %(self.__nHeis, self.nBaud))
620 626
621 627 #Frequency
622 628 __codeBuffer = numpy.zeros((self.nCode, self.__nHeis), dtype=numpy.complex)
623 629
624 630 __codeBuffer[:,0:self.nBaud] = self.code
625 631
626 632 self.fft_code = numpy.conj(numpy.fft.fft(__codeBuffer, axis=1))
627 633
628 634 if dataOut.flagDataAsBlock:
629 635
630 636 self.ndatadec = self.__nHeis #- self.nBaud + 1
631 637
632 638 self.datadecTime = numpy.zeros((self.__nChannels, self.__nProfiles, self.ndatadec), dtype=numpy.complex)
633 639
634 640 else:
635 641
636 642 #Time
637 643 self.ndatadec = self.__nHeis #- self.nBaud + 1
638 644
639 645 self.datadecTime = numpy.zeros((self.__nChannels, self.ndatadec), dtype=numpy.complex)
640 646
641 647 def __convolutionInFreq(self, data):
642 648
643 649 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
644 650
645 651 fft_data = numpy.fft.fft(data, axis=1)
646 652
647 653 conv = fft_data*fft_code
648 654
649 655 data = numpy.fft.ifft(conv,axis=1)
650 656
651 657 return data
652 658
653 659 def __convolutionInFreqOpt(self, data):
654 660
655 661 raise NotImplementedError
656 662
657 663 def __convolutionInTime(self, data):
658 664
659 665 code = self.code[self.__profIndex]
660 666 for i in range(self.__nChannels):
661 667 self.datadecTime[i,:] = numpy.correlate(data[i,:], code, mode='full')[self.nBaud-1:]
662 668
663 669 return self.datadecTime
664 670
665 671 def __convolutionByBlockInTime(self, data):
666 672
667 673 repetitions = self.__nProfiles / self.nCode
668 674
669 675 junk = numpy.lib.stride_tricks.as_strided(self.code, (repetitions, self.code.size), (0, self.code.itemsize))
670 676 junk = junk.flatten()
671 677 code_block = numpy.reshape(junk, (self.nCode*repetitions, self.nBaud))
672 678 profilesList = range(self.__nProfiles)
673 679
674 680 for i in range(self.__nChannels):
675 681 for j in profilesList:
676 682 self.datadecTime[i,j,:] = numpy.correlate(data[i,j,:], code_block[j,:], mode='full')[self.nBaud-1:]
677 683 return self.datadecTime
678 684
679 685 def __convolutionByBlockInFreq(self, data):
680 686
681 687 raise NotImplementedError("Decoder by frequency fro Blocks not implemented")
682 688
683 689
684 690 fft_code = self.fft_code[self.__profIndex].reshape(1,-1)
685 691
686 692 fft_data = numpy.fft.fft(data, axis=2)
687 693
688 694 conv = fft_data*fft_code
689 695
690 696 data = numpy.fft.ifft(conv,axis=2)
691 697
692 698 return data
693 699
694 700
695 701 def run(self, dataOut, code=None, nCode=None, nBaud=None, mode = 0, osamp=None, times=None):
696 702
697 703 if dataOut.flagDecodeData:
698 704 print("This data is already decoded, recoding again ...")
699 705
700 706 if not self.isConfig:
701 707
702 708 if code is None:
703 709 if dataOut.code is None:
704 710 raise ValueError("Code could not be read from %s instance. Enter a value in Code parameter" %dataOut.type)
705 711
706 712 code = dataOut.code
707 713 else:
708 714 code = numpy.array(code).reshape(nCode,nBaud)
709 715 self.setup(code, osamp, dataOut)
710 716
711 717 self.isConfig = True
712 718
713 719 if mode == 3:
714 720 sys.stderr.write("Decoder Warning: mode=%d is not valid, using mode=0\n" %mode)
715 721
716 722 if times != None:
717 723 sys.stderr.write("Decoder Warning: Argument 'times' in not used anymore\n")
718 724
719 725 if self.code is None:
720 726 print("Fail decoding: Code is not defined.")
721 727 return
722 728
723 729 self.__nProfiles = dataOut.nProfiles
724 730 datadec = None
725 731
726 732 if mode == 3:
727 733 mode = 0
728 734
729 735 if dataOut.flagDataAsBlock:
730 736 """
731 737 Decoding when data have been read as block,
732 738 """
733 739
734 740 if mode == 0:
735 741 datadec = self.__convolutionByBlockInTime(dataOut.data)
736 742 if mode == 1:
737 743 datadec = self.__convolutionByBlockInFreq(dataOut.data)
738 744 else:
739 745 """
740 746 Decoding when data have been read profile by profile
741 747 """
742 748 if mode == 0:
743 749 datadec = self.__convolutionInTime(dataOut.data)
744 750
745 751 if mode == 1:
746 752 datadec = self.__convolutionInFreq(dataOut.data)
747 753
748 754 if mode == 2:
749 755 datadec = self.__convolutionInFreqOpt(dataOut.data)
750 756
751 757 if datadec is None:
752 758 raise ValueError("Codification mode selected is not valid: mode=%d. Try selecting 0 or 1" %mode)
753 759
754 760 dataOut.code = self.code
755 761 dataOut.nCode = self.nCode
756 762 dataOut.nBaud = self.nBaud
757 763
758 764 dataOut.data = datadec
759 765
760 766 dataOut.heightList = dataOut.heightList[0:datadec.shape[-1]]
761 767
762 768 dataOut.flagDecodeData = True #asumo q la data esta decodificada
763 769
764 770 if self.__profIndex == self.nCode-1:
765 771 self.__profIndex = 0
766 return 1
772 return dataOut
767 773
768 774 self.__profIndex += 1
769 775
770 return 1
776 return dataOut
771 777 # dataOut.flagDeflipData = True #asumo q la data no esta sin flip
772 778
773
779 @MPDecorator
774 780 class ProfileConcat(Operation):
775 781
776 782 isConfig = False
777 783 buffer = None
778 784
779 def __init__(self, **kwargs):
785 def __init__(self):#, **kwargs):
780 786
781 Operation.__init__(self, **kwargs)
787 Operation.__init__(self)#, **kwargs)
782 788 self.profileIndex = 0
783 789
784 790 def reset(self):
785 791 self.buffer = numpy.zeros_like(self.buffer)
786 792 self.start_index = 0
787 793 self.times = 1
788 794
789 795 def setup(self, data, m, n=1):
790 796 self.buffer = numpy.zeros((data.shape[0],data.shape[1]*m),dtype=type(data[0,0]))
791 797 self.nHeights = data.shape[1]#.nHeights
792 798 self.start_index = 0
793 799 self.times = 1
794 800
795 801 def concat(self, data):
796 802
797 803 self.buffer[:,self.start_index:self.nHeights*self.times] = data.copy()
798 804 self.start_index = self.start_index + self.nHeights
799 805
800 806 def run(self, dataOut, m):
801 807
802 808 dataOut.flagNoData = True
803 809
804 810 if not self.isConfig:
805 811 self.setup(dataOut.data, m, 1)
806 812 self.isConfig = True
807 813
808 814 if dataOut.flagDataAsBlock:
809 815 raise ValueError("ProfileConcat can only be used when voltage have been read profile by profile, getBlock = False")
810 816
811 817 else:
812 818 self.concat(dataOut.data)
813 819 self.times += 1
814 820 if self.times > m:
815 821 dataOut.data = self.buffer
816 822 self.reset()
817 823 dataOut.flagNoData = False
818 824 # se deben actualizar mas propiedades del header y del objeto dataOut, por ejemplo, las alturas
819 825 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
820 826 xf = dataOut.heightList[0] + dataOut.nHeights * deltaHeight * m
821 827 dataOut.heightList = numpy.arange(dataOut.heightList[0], xf, deltaHeight)
822 828 dataOut.ippSeconds *= m
823
829 return dataOut
830 @MPDecorator
824 831 class ProfileSelector(Operation):
825 832
826 833 profileIndex = None
827 834 # Tamanho total de los perfiles
828 835 nProfiles = None
829 836
830 def __init__(self, **kwargs):
837 def __init__(self):#, **kwargs):
831 838
832 Operation.__init__(self, **kwargs)
839 Operation.__init__(self)#, **kwargs)
833 840 self.profileIndex = 0
834 841
835 842 def incProfileIndex(self):
836 843
837 844 self.profileIndex += 1
838 845
839 846 if self.profileIndex >= self.nProfiles:
840 847 self.profileIndex = 0
841 848
842 849 def isThisProfileInRange(self, profileIndex, minIndex, maxIndex):
843 850
844 851 if profileIndex < minIndex:
845 852 return False
846 853
847 854 if profileIndex > maxIndex:
848 855 return False
849 856
850 857 return True
851 858
852 859 def isThisProfileInList(self, profileIndex, profileList):
853 860
854 861 if profileIndex not in profileList:
855 862 return False
856 863
857 864 return True
858 865
859 866 def run(self, dataOut, profileList=None, profileRangeList=None, beam=None, byblock=False, rangeList = None, nProfiles=None):
860 867
861 868 """
862 869 ProfileSelector:
863 870
864 871 Inputs:
865 872 profileList : Index of profiles selected. Example: profileList = (0,1,2,7,8)
866 873
867 874 profileRangeList : Minimum and maximum profile indexes. Example: profileRangeList = (4, 30)
868 875
869 876 rangeList : List of profile ranges. Example: rangeList = ((4, 30), (32, 64), (128, 256))
870 877
871 878 """
872 879
873 880 if rangeList is not None:
874 881 if type(rangeList[0]) not in (tuple, list):
875 882 rangeList = [rangeList]
876 883
877 884 dataOut.flagNoData = True
878 885
879 886 if dataOut.flagDataAsBlock:
880 887 """
881 888 data dimension = [nChannels, nProfiles, nHeis]
882 889 """
883 890 if profileList != None:
884 891 dataOut.data = dataOut.data[:,profileList,:]
885 892
886 893 if profileRangeList != None:
887 894 minIndex = profileRangeList[0]
888 895 maxIndex = profileRangeList[1]
889 896 profileList = list(range(minIndex, maxIndex+1))
890 897
891 898 dataOut.data = dataOut.data[:,minIndex:maxIndex+1,:]
892 899
893 900 if rangeList != None:
894 901
895 902 profileList = []
896 903
897 904 for thisRange in rangeList:
898 905 minIndex = thisRange[0]
899 906 maxIndex = thisRange[1]
900 907
901 908 profileList.extend(list(range(minIndex, maxIndex+1)))
902 909
903 910 dataOut.data = dataOut.data[:,profileList,:]
904 911
905 912 dataOut.nProfiles = len(profileList)
906 913 dataOut.profileIndex = dataOut.nProfiles - 1
907 914 dataOut.flagNoData = False
908 915
909 916 return True
910 917
911 918 """
912 919 data dimension = [nChannels, nHeis]
913 920 """
914 921
915 922 if profileList != None:
916 923
917 924 if self.isThisProfileInList(dataOut.profileIndex, profileList):
918 925
919 926 self.nProfiles = len(profileList)
920 927 dataOut.nProfiles = self.nProfiles
921 928 dataOut.profileIndex = self.profileIndex
922 929 dataOut.flagNoData = False
923 930
924 931 self.incProfileIndex()
925 932 return True
926 933
927 934 if profileRangeList != None:
928 935
929 936 minIndex = profileRangeList[0]
930 937 maxIndex = profileRangeList[1]
931 938
932 939 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
933 940
934 941 self.nProfiles = maxIndex - minIndex + 1
935 942 dataOut.nProfiles = self.nProfiles
936 943 dataOut.profileIndex = self.profileIndex
937 944 dataOut.flagNoData = False
938 945
939 946 self.incProfileIndex()
940 947 return True
941 948
942 949 if rangeList != None:
943 950
944 951 nProfiles = 0
945 952
946 953 for thisRange in rangeList:
947 954 minIndex = thisRange[0]
948 955 maxIndex = thisRange[1]
949 956
950 957 nProfiles += maxIndex - minIndex + 1
951 958
952 959 for thisRange in rangeList:
953 960
954 961 minIndex = thisRange[0]
955 962 maxIndex = thisRange[1]
956 963
957 964 if self.isThisProfileInRange(dataOut.profileIndex, minIndex, maxIndex):
958 965
959 966 self.nProfiles = nProfiles
960 967 dataOut.nProfiles = self.nProfiles
961 968 dataOut.profileIndex = self.profileIndex
962 969 dataOut.flagNoData = False
963 970
964 971 self.incProfileIndex()
965 972
966 973 break
967 974
968 975 return True
969 976
970 977
971 978 if beam != None: #beam is only for AMISR data
972 979 if self.isThisProfileInList(dataOut.profileIndex, dataOut.beamRangeDict[beam]):
973 980 dataOut.flagNoData = False
974 981 dataOut.profileIndex = self.profileIndex
975 982
976 983 self.incProfileIndex()
977 984
978 985 return True
979 986
980 987 raise ValueError("ProfileSelector needs profileList, profileRangeList or rangeList parameter")
981 988
982 return False
983
989 #return False
990 return dataOut
991 @MPDecorator
984 992 class Reshaper(Operation):
985 993
986 def __init__(self, **kwargs):
994 def __init__(self):#, **kwargs):
987 995
988 Operation.__init__(self, **kwargs)
996 Operation.__init__(self)#, **kwargs)
989 997
990 998 self.__buffer = None
991 999 self.__nitems = 0
992 1000
993 1001 def __appendProfile(self, dataOut, nTxs):
994 1002
995 1003 if self.__buffer is None:
996 1004 shape = (dataOut.nChannels, int(dataOut.nHeights/nTxs) )
997 1005 self.__buffer = numpy.empty(shape, dtype = dataOut.data.dtype)
998 1006
999 1007 ini = dataOut.nHeights * self.__nitems
1000 1008 end = ini + dataOut.nHeights
1001 1009
1002 1010 self.__buffer[:, ini:end] = dataOut.data
1003 1011
1004 1012 self.__nitems += 1
1005 1013
1006 1014 return int(self.__nitems*nTxs)
1007 1015
1008 1016 def __getBuffer(self):
1009 1017
1010 1018 if self.__nitems == int(1./self.__nTxs):
1011 1019
1012 1020 self.__nitems = 0
1013 1021
1014 1022 return self.__buffer.copy()
1015 1023
1016 1024 return None
1017 1025
1018 1026 def __checkInputs(self, dataOut, shape, nTxs):
1019 1027
1020 1028 if shape is None and nTxs is None:
1021 1029 raise ValueError("Reshaper: shape of factor should be defined")
1022 1030
1023 1031 if nTxs:
1024 1032 if nTxs < 0:
1025 1033 raise ValueError("nTxs should be greater than 0")
1026 1034
1027 1035 if nTxs < 1 and dataOut.nProfiles % (1./nTxs) != 0:
1028 1036 raise ValueError("nProfiles= %d is not divisibled by (1./nTxs) = %f" %(dataOut.nProfiles, (1./nTxs)))
1029 1037
1030 1038 shape = [dataOut.nChannels, dataOut.nProfiles*nTxs, dataOut.nHeights/nTxs]
1031 1039
1032 1040 return shape, nTxs
1033 1041
1034 1042 if len(shape) != 2 and len(shape) != 3:
1035 1043 raise ValueError("shape dimension should be equal to 2 or 3. shape = (nProfiles, nHeis) or (nChannels, nProfiles, nHeis). Actually shape = (%d, %d, %d)" %(dataOut.nChannels, dataOut.nProfiles, dataOut.nHeights))
1036 1044
1037 1045 if len(shape) == 2:
1038 1046 shape_tuple = [dataOut.nChannels]
1039 1047 shape_tuple.extend(shape)
1040 1048 else:
1041 1049 shape_tuple = list(shape)
1042 1050
1043 1051 nTxs = 1.0*shape_tuple[1]/dataOut.nProfiles
1044 1052
1045 1053 return shape_tuple, nTxs
1046 1054
1047 1055 def run(self, dataOut, shape=None, nTxs=None):
1048 1056
1049 1057 shape_tuple, self.__nTxs = self.__checkInputs(dataOut, shape, nTxs)
1050 1058
1051 1059 dataOut.flagNoData = True
1052 1060 profileIndex = None
1053 1061
1054 1062 if dataOut.flagDataAsBlock:
1055 1063
1056 1064 dataOut.data = numpy.reshape(dataOut.data, shape_tuple)
1057 1065 dataOut.flagNoData = False
1058 1066
1059 1067 profileIndex = int(dataOut.nProfiles*self.__nTxs) - 1
1060 1068
1061 1069 else:
1062 1070
1063 1071 if self.__nTxs < 1:
1064 1072
1065 1073 self.__appendProfile(dataOut, self.__nTxs)
1066 1074 new_data = self.__getBuffer()
1067 1075
1068 1076 if new_data is not None:
1069 1077 dataOut.data = new_data
1070 1078 dataOut.flagNoData = False
1071 1079
1072 1080 profileIndex = dataOut.profileIndex*nTxs
1073 1081
1074 1082 else:
1075 1083 raise ValueError("nTxs should be greater than 0 and lower than 1, or use VoltageReader(..., getblock=True)")
1076 1084
1077 1085 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1078 1086
1079 1087 dataOut.heightList = numpy.arange(dataOut.nHeights/self.__nTxs) * deltaHeight + dataOut.heightList[0]
1080 1088
1081 1089 dataOut.nProfiles = int(dataOut.nProfiles*self.__nTxs)
1082 1090
1083 1091 dataOut.profileIndex = profileIndex
1084 1092
1085 1093 dataOut.ippSeconds /= self.__nTxs
1086 1094
1095 return dataOut
1096 @MPDecorator
1087 1097 class SplitProfiles(Operation):
1088 1098
1089 def __init__(self, **kwargs):
1099 def __init__(self):#, **kwargs):
1090 1100
1091 Operation.__init__(self, **kwargs)
1101 Operation.__init__(self)#, **kwargs)
1092 1102
1093 1103 def run(self, dataOut, n):
1094 1104
1095 1105 dataOut.flagNoData = True
1096 1106 profileIndex = None
1097 1107
1098 1108 if dataOut.flagDataAsBlock:
1099 1109
1100 1110 #nchannels, nprofiles, nsamples
1101 1111 shape = dataOut.data.shape
1102 1112
1103 1113 if shape[2] % n != 0:
1104 1114 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[2]))
1105
1115
1106 1116 new_shape = shape[0], shape[1]*n, int(shape[2]/n)
1117
1107 1118 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1108 1119 dataOut.flagNoData = False
1109 1120
1110 1121 profileIndex = int(dataOut.nProfiles/n) - 1
1111 1122
1112 1123 else:
1113 1124
1114 1125 raise ValueError("Could not split the data when is read Profile by Profile. Use VoltageReader(..., getblock=True)")
1115 1126
1116 1127 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1117 1128
1118 1129 dataOut.heightList = numpy.arange(dataOut.nHeights/n) * deltaHeight + dataOut.heightList[0]
1119 1130
1120 1131 dataOut.nProfiles = int(dataOut.nProfiles*n)
1121 1132
1122 1133 dataOut.profileIndex = profileIndex
1123 1134
1124 1135 dataOut.ippSeconds /= n
1125 1136
1137 return dataOut
1138 @MPDecorator
1126 1139 class CombineProfiles(Operation):
1127 def __init__(self, **kwargs):
1140 def __init__(self):#, **kwargs):
1128 1141
1129 Operation.__init__(self, **kwargs)
1142 Operation.__init__(self)#, **kwargs)
1130 1143
1131 1144 self.__remData = None
1132 1145 self.__profileIndex = 0
1133 1146
1134 1147 def run(self, dataOut, n):
1135 1148
1136 1149 dataOut.flagNoData = True
1137 1150 profileIndex = None
1138 1151
1139 1152 if dataOut.flagDataAsBlock:
1140 1153
1141 1154 #nchannels, nprofiles, nsamples
1142 1155 shape = dataOut.data.shape
1143 1156 new_shape = shape[0], shape[1]/n, shape[2]*n
1144 1157
1145 1158 if shape[1] % n != 0:
1146 1159 raise ValueError("Could not split the data, n=%d has to be multiple of %d" %(n, shape[1]))
1147 1160
1148 1161 dataOut.data = numpy.reshape(dataOut.data, new_shape)
1149 1162 dataOut.flagNoData = False
1150 1163
1151 1164 profileIndex = int(dataOut.nProfiles*n) - 1
1152 1165
1153 1166 else:
1154 1167
1155 1168 #nchannels, nsamples
1156 1169 if self.__remData is None:
1157 1170 newData = dataOut.data
1158 1171 else:
1159 1172 newData = numpy.concatenate((self.__remData, dataOut.data), axis=1)
1160 1173
1161 1174 self.__profileIndex += 1
1162 1175
1163 1176 if self.__profileIndex < n:
1164 1177 self.__remData = newData
1165 1178 #continue
1166 1179 return
1167 1180
1168 1181 self.__profileIndex = 0
1169 1182 self.__remData = None
1170 1183
1171 1184 dataOut.data = newData
1172 1185 dataOut.flagNoData = False
1173 1186
1174 1187 profileIndex = dataOut.profileIndex/n
1175 1188
1176 1189
1177 1190 deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1178 1191
1179 1192 dataOut.heightList = numpy.arange(dataOut.nHeights*n) * deltaHeight + dataOut.heightList[0]
1180 1193
1181 1194 dataOut.nProfiles = int(dataOut.nProfiles/n)
1182 1195
1183 1196 dataOut.profileIndex = profileIndex
1184 1197
1185 1198 dataOut.ippSeconds *= n
1186 1199
1200 return dataOut
1187 1201 # import collections
1188 1202 # from scipy.stats import mode
1189 1203 #
1190 1204 # class Synchronize(Operation):
1191 1205 #
1192 1206 # isConfig = False
1193 1207 # __profIndex = 0
1194 1208 #
1195 1209 # def __init__(self, **kwargs):
1196 1210 #
1197 1211 # Operation.__init__(self, **kwargs)
1198 1212 # # self.isConfig = False
1199 1213 # self.__powBuffer = None
1200 1214 # self.__startIndex = 0
1201 1215 # self.__pulseFound = False
1202 1216 #
1203 1217 # def __findTxPulse(self, dataOut, channel=0, pulse_with = None):
1204 1218 #
1205 1219 # #Read data
1206 1220 #
1207 1221 # powerdB = dataOut.getPower(channel = channel)
1208 1222 # noisedB = dataOut.getNoise(channel = channel)[0]
1209 1223 #
1210 1224 # self.__powBuffer.extend(powerdB.flatten())
1211 1225 #
1212 1226 # dataArray = numpy.array(self.__powBuffer)
1213 1227 #
1214 1228 # filteredPower = numpy.correlate(dataArray, dataArray[0:self.__nSamples], "same")
1215 1229 #
1216 1230 # maxValue = numpy.nanmax(filteredPower)
1217 1231 #
1218 1232 # if maxValue < noisedB + 10:
1219 1233 # #No se encuentra ningun pulso de transmision
1220 1234 # return None
1221 1235 #
1222 1236 # maxValuesIndex = numpy.where(filteredPower > maxValue - 0.1*abs(maxValue))[0]
1223 1237 #
1224 1238 # if len(maxValuesIndex) < 2:
1225 1239 # #Solo se encontro un solo pulso de transmision de un baudio, esperando por el siguiente TX
1226 1240 # return None
1227 1241 #
1228 1242 # phasedMaxValuesIndex = maxValuesIndex - self.__nSamples
1229 1243 #
1230 1244 # #Seleccionar solo valores con un espaciamiento de nSamples
1231 1245 # pulseIndex = numpy.intersect1d(maxValuesIndex, phasedMaxValuesIndex)
1232 1246 #
1233 1247 # if len(pulseIndex) < 2:
1234 1248 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1235 1249 # return None
1236 1250 #
1237 1251 # spacing = pulseIndex[1:] - pulseIndex[:-1]
1238 1252 #
1239 1253 # #remover senales que se distancien menos de 10 unidades o muestras
1240 1254 # #(No deberian existir IPP menor a 10 unidades)
1241 1255 #
1242 1256 # realIndex = numpy.where(spacing > 10 )[0]
1243 1257 #
1244 1258 # if len(realIndex) < 2:
1245 1259 # #Solo se encontro un pulso de transmision con ancho mayor a 1
1246 1260 # return None
1247 1261 #
1248 1262 # #Eliminar pulsos anchos (deja solo la diferencia entre IPPs)
1249 1263 # realPulseIndex = pulseIndex[realIndex]
1250 1264 #
1251 1265 # period = mode(realPulseIndex[1:] - realPulseIndex[:-1])[0][0]
1252 1266 #
1253 1267 # print "IPP = %d samples" %period
1254 1268 #
1255 1269 # self.__newNSamples = dataOut.nHeights #int(period)
1256 1270 # self.__startIndex = int(realPulseIndex[0])
1257 1271 #
1258 1272 # return 1
1259 1273 #
1260 1274 #
1261 1275 # def setup(self, nSamples, nChannels, buffer_size = 4):
1262 1276 #
1263 1277 # self.__powBuffer = collections.deque(numpy.zeros( buffer_size*nSamples,dtype=numpy.float),
1264 1278 # maxlen = buffer_size*nSamples)
1265 1279 #
1266 1280 # bufferList = []
1267 1281 #
1268 1282 # for i in range(nChannels):
1269 1283 # bufferByChannel = collections.deque(numpy.zeros( buffer_size*nSamples, dtype=numpy.complex) + numpy.NAN,
1270 1284 # maxlen = buffer_size*nSamples)
1271 1285 #
1272 1286 # bufferList.append(bufferByChannel)
1273 1287 #
1274 1288 # self.__nSamples = nSamples
1275 1289 # self.__nChannels = nChannels
1276 1290 # self.__bufferList = bufferList
1277 1291 #
1278 1292 # def run(self, dataOut, channel = 0):
1279 1293 #
1280 1294 # if not self.isConfig:
1281 1295 # nSamples = dataOut.nHeights
1282 1296 # nChannels = dataOut.nChannels
1283 1297 # self.setup(nSamples, nChannels)
1284 1298 # self.isConfig = True
1285 1299 #
1286 1300 # #Append new data to internal buffer
1287 1301 # for thisChannel in range(self.__nChannels):
1288 1302 # bufferByChannel = self.__bufferList[thisChannel]
1289 1303 # bufferByChannel.extend(dataOut.data[thisChannel])
1290 1304 #
1291 1305 # if self.__pulseFound:
1292 1306 # self.__startIndex -= self.__nSamples
1293 1307 #
1294 1308 # #Finding Tx Pulse
1295 1309 # if not self.__pulseFound:
1296 1310 # indexFound = self.__findTxPulse(dataOut, channel)
1297 1311 #
1298 1312 # if indexFound == None:
1299 1313 # dataOut.flagNoData = True
1300 1314 # return
1301 1315 #
1302 1316 # self.__arrayBuffer = numpy.zeros((self.__nChannels, self.__newNSamples), dtype = numpy.complex)
1303 1317 # self.__pulseFound = True
1304 1318 # self.__startIndex = indexFound
1305 1319 #
1306 1320 # #If pulse was found ...
1307 1321 # for thisChannel in range(self.__nChannels):
1308 1322 # bufferByChannel = self.__bufferList[thisChannel]
1309 1323 # #print self.__startIndex
1310 1324 # x = numpy.array(bufferByChannel)
1311 1325 # self.__arrayBuffer[thisChannel] = x[self.__startIndex:self.__startIndex+self.__newNSamples]
1312 1326 #
1313 1327 # deltaHeight = dataOut.heightList[1] - dataOut.heightList[0]
1314 1328 # dataOut.heightList = numpy.arange(self.__newNSamples)*deltaHeight
1315 1329 # # dataOut.ippSeconds = (self.__newNSamples / deltaHeight)/1e6
1316 1330 #
1317 1331 # dataOut.data = self.__arrayBuffer
1318 1332 #
1319 1333 # self.__startIndex += self.__newNSamples
1320 1334 #
1321 # return No newline at end of file
1335 # return
General Comments 0
You need to be logged in to leave comments. Login now