##// END OF EJS Templates
schain-jc
RSS
merge con digital_rf y pull
José Chávez -
r1067:7abdb8296c68 merge
parent child
Show More
Side by Side Unified
Context file:
r1067:7abdb8296c68 Loading diff...:

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1324 +1,1297
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 from multiprocessing import Process, Queue, cpu_count
13
14 import schainpy
15 import schainpy.admin
16 from schainpy.utils.log import logToFile
12 from multiprocessing import Process, cpu_count
17 13
18 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 15 from xml.dom import minidom
20 16
17 import schainpy
18 import schainpy.admin
21 19 from schainpy.model import *
22 from time import sleep
23
24
25
26 def prettify(elem):
27 """Return a pretty-printed XML string for the Element.
28 """
29 rough_string = tostring(elem, 'utf-8')
30 reparsed = minidom.parseString(rough_string)
31 return reparsed.toprettyxml(indent=" ")
32
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 skip = 0
35 cursor = 0
36 nFiles = None
37 processes = []
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
20 from schainpy.utils import log
21
22 DTYPES = {
23 'Voltage': '.r',
24 'Spectra': '.pdata'
25 }
26
27 def MPProject(project, n=cpu_count()):
28 '''
29 Project wrapper to run schain in n processes
30 '''
31
32 rconf = project.getReadUnitObj()
33 op = rconf.getOperationObj('run')
34 dt1 = op.getParameterValue('startDate')
35 dt2 = op.getParameterValue('endDate')
40 36 days = (dt2 - dt1).days
41
37
42 38 for day in range(days+1):
43 39 skip = 0
44 40 cursor = 0
45 q = Queue()
46 41 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
49 firstProcess.start()
50 if by_day:
51 continue
52 nFiles = q.get()
53 if nFiles==0:
42 dt = dt1 + datetime.timedelta(day)
43 dt_str = dt.strftime('%Y/%m/%d')
44 reader = JRODataReader()
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 startDate=dt,
47 endDate=dt,
48 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
50 if nFiles == 0:
54 51 continue
55 firstProcess.terminate()
56 skip = int(math.ceil(nFiles/nProcess))
57 while True:
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
59 processes[cursor].start()
60 if nFiles < cursor*skip:
61 break
52 skip = int(math.ceil(nFiles/n))
53 while nFiles > cursor*skip:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 skip=skip)
56 p = project.clone()
57 p.start()
58 processes.append(p)
62 59 cursor += 1
63 60
64 61 def beforeExit(exctype, value, trace):
65 62 for process in processes:
66 63 process.terminate()
67 64 process.join()
68 65 print traceback.print_tb(trace)
69 66
70 67 sys.excepthook = beforeExit
71 68
72 69 for process in processes:
73 70 process.join()
74 71 process.terminate()
75 72
76 73 time.sleep(3)
77 74
78
79 75 class ParameterConf():
80 76
81 77 id = None
82 78 name = None
83 79 value = None
84 80 format = None
85 81
86 82 __formated_value = None
87 83
88 84 ELEMENTNAME = 'Parameter'
89 85
90 86 def __init__(self):
91 87
92 88 self.format = 'str'
93 89
94 90 def getElementName(self):
95 91
96 92 return self.ELEMENTNAME
97 93
98 94 def getValue(self):
99 95
100 96 value = self.value
101 97 format = self.format
102 98
103 99 if self.__formated_value != None:
104 100
105 101 return self.__formated_value
106 102
107 103 if format == 'obj':
108 104 return value
109 105
110 106 if format == 'str':
111 107 self.__formated_value = str(value)
112 108 return self.__formated_value
113 109
114 110 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
111 raise ValueError, '%s: This parameter value is empty' %self.name
116 112
117 113 if format == 'list':
118 114 strList = value.split(',')
119 115
120 116 self.__formated_value = strList
121 117
122 118 return self.__formated_value
123 119
124 120 if format == 'intlist':
125 """
121 '''
126 122 Example:
127 123 value = (0,1,2)
128 """
124 '''
129 125
130 126 new_value = ast.literal_eval(value)
131 127
132 128 if type(new_value) not in (tuple, list):
133 129 new_value = [int(new_value)]
134 130
135 131 self.__formated_value = new_value
136 132
137 133 return self.__formated_value
138 134
139 135 if format == 'floatlist':
140 """
136 '''
141 137 Example:
142 138 value = (0.5, 1.4, 2.7)
143 """
139 '''
144 140
145 141 new_value = ast.literal_eval(value)
146 142
147 143 if type(new_value) not in (tuple, list):
148 144 new_value = [float(new_value)]
149 145
150 146 self.__formated_value = new_value
151 147
152 148 return self.__formated_value
153 149
154 150 if format == 'date':
155 151 strList = value.split('/')
156 152 intList = [int(x) for x in strList]
157 153 date = datetime.date(intList[0], intList[1], intList[2])
158 154
159 155 self.__formated_value = date
160 156
161 157 return self.__formated_value
162 158
163 159 if format == 'time':
164 160 strList = value.split(':')
165 161 intList = [int(x) for x in strList]
166 162 time = datetime.time(intList[0], intList[1], intList[2])
167 163
168 164 self.__formated_value = time
169 165
170 166 return self.__formated_value
171 167
172 168 if format == 'pairslist':
173 """
169 '''
174 170 Example:
175 171 value = (0,1),(1,2)
176 """
172 '''
177 173
178 174 new_value = ast.literal_eval(value)
179 175
180 176 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
182 178
183 179 if type(new_value[0]) not in (tuple, list):
184 180 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
186 182 new_value = [new_value]
187 183
188 184 for thisPair in new_value:
189 185 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
191 187
192 188 self.__formated_value = new_value
193 189
194 190 return self.__formated_value
195 191
196 192 if format == 'multilist':
197 """
193 '''
198 194 Example:
199 195 value = (0,1,2),(3,4,5)
200 """
196 '''
201 197 multiList = ast.literal_eval(value)
202 198
203 199 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
200 multiList = ast.literal_eval('(' + value + ')')
205 201
206 202 self.__formated_value = multiList
207 203
208 204 return self.__formated_value
209 205
210 206 if format == 'bool':
211 207 value = int(value)
212 208
213 209 if format == 'int':
214 210 value = float(value)
215 211
216 212 format_func = eval(format)
217 213
218 214 self.__formated_value = format_func(value)
219 215
220 216 return self.__formated_value
221 217
222 218 def updateId(self, new_id):
223 219
224 220 self.id = str(new_id)
225 221
226 222 def setup(self, id, name, value, format='str'):
227 223 self.id = str(id)
228 224 self.name = name
229 225 if format == 'obj':
230 226 self.value = value
231 227 else:
232 228 self.value = str(value)
233 229 self.format = str.lower(format)
234 230
235 231 self.getValue()
236 232
237 233 return 1
238 234
239 235 def update(self, name, value, format='str'):
240 236
241 237 self.name = name
242 238 self.value = str(value)
243 239 self.format = format
244 240
245 241 def makeXml(self, opElement):
246 242 if self.name not in ('queue',):
247 243 parmElement = SubElement(opElement, self.ELEMENTNAME)
248 244 parmElement.set('id', str(self.id))
249 245 parmElement.set('name', self.name)
250 246 parmElement.set('value', self.value)
251 247 parmElement.set('format', self.format)
252 248
253 249 def readXml(self, parmElement):
254 250
255 251 self.id = parmElement.get('id')
256 252 self.name = parmElement.get('name')
257 253 self.value = parmElement.get('value')
258 254 self.format = str.lower(parmElement.get('format'))
259 255
260 256 #Compatible with old signal chain version
261 257 if self.format == 'int' and self.name == 'idfigure':
262 258 self.name = 'id'
263 259
264 260 def printattr(self):
265 261
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
267 263
268 class OperationConf():
264 class OperationConf():
269 265
270 266 id = None
271 267 name = None
272 268 priority = None
273 269 type = None
274 270
275 271 parmConfObjList = []
276 272
277 273 ELEMENTNAME = 'Operation'
278 274
279 275 def __init__(self):
280 276
281 277 self.id = '0'
282 278 self.name = None
283 279 self.priority = None
284 280 self.type = 'self'
285 281
286 282
287 283 def __getNewId(self):
288 284
289 285 return int(self.id)*10 + len(self.parmConfObjList) + 1
290 286
291 287 def updateId(self, new_id):
292 288
293 289 self.id = str(new_id)
294 290
295 291 n = 1
296 292 for parmObj in self.parmConfObjList:
297 293
298 294 idParm = str(int(new_id)*10 + n)
299 295 parmObj.updateId(idParm)
300 296
301 297 n += 1
302 298
303 299 def getElementName(self):
304 300
305 301 return self.ELEMENTNAME
306 302
307 303 def getParameterObjList(self):
308 304
309 305 return self.parmConfObjList
310 306
311 307 def getParameterObj(self, parameterName):
312 308
313 309 for parmConfObj in self.parmConfObjList:
314 310
315 311 if parmConfObj.name != parameterName:
316 312 continue
317 313
318 314 return parmConfObj
319 315
320 316 return None
321 317
322 318 def getParameterObjfromValue(self, parameterValue):
323 319
324 320 for parmConfObj in self.parmConfObjList:
325 321
326 322 if parmConfObj.getValue() != parameterValue:
327 323 continue
328 324
329 325 return parmConfObj.getValue()
330 326
331 327 return None
332 328
333 329 def getParameterValue(self, parameterName):
334 330
335 331 parameterObj = self.getParameterObj(parameterName)
336 332
337 333 # if not parameterObj:
338 334 # return None
339 335
340 336 value = parameterObj.getValue()
341 337
342 338 return value
343 339
344 340
345 341 def getKwargs(self):
346 342
347 343 kwargs = {}
348 344
349 345 for parmConfObj in self.parmConfObjList:
350 346 if self.name == 'run' and parmConfObj.name == 'datatype':
351 347 continue
352 348
353 349 kwargs[parmConfObj.name] = parmConfObj.getValue()
354 350
355 351 return kwargs
356 352
357 353 def setup(self, id, name, priority, type):
358 354
359 355 self.id = str(id)
360 356 self.name = name
361 357 self.type = type
362 358 self.priority = priority
363 359
364 360 self.parmConfObjList = []
365 361
366 362 def removeParameters(self):
367 363
368 364 for obj in self.parmConfObjList:
369 365 del obj
370 366
371 367 self.parmConfObjList = []
372 368
373 369 def addParameter(self, name, value, format='str'):
374
370
371 if value is None:
372 return None
375 373 id = self.__getNewId()
376 374
377 375 parmConfObj = ParameterConf()
378 376 if not parmConfObj.setup(id, name, value, format):
379 377 return None
380 378
381 379 self.parmConfObjList.append(parmConfObj)
382 380
383 381 return parmConfObj
384 382
385 383 def changeParameter(self, name, value, format='str'):
386 384
387 385 parmConfObj = self.getParameterObj(name)
388 386 parmConfObj.update(name, value, format)
389 387
390 388 return parmConfObj
391 389
392 390 def makeXml(self, procUnitElement):
393 391
394 392 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
395 393 opElement.set('id', str(self.id))
396 394 opElement.set('name', self.name)
397 395 opElement.set('type', self.type)
398 396 opElement.set('priority', str(self.priority))
399 397
400 398 for parmConfObj in self.parmConfObjList:
401 399 parmConfObj.makeXml(opElement)
402 400
403 401 def readXml(self, opElement):
404 402
405 403 self.id = opElement.get('id')
406 404 self.name = opElement.get('name')
407 405 self.type = opElement.get('type')
408 406 self.priority = opElement.get('priority')
409 407
410 408 #Compatible with old signal chain version
411 409 #Use of 'run' method instead 'init'
412 410 if self.type == 'self' and self.name == 'init':
413 411 self.name = 'run'
414 412
415 413 self.parmConfObjList = []
416 414
417 415 parmElementList = opElement.iter(ParameterConf().getElementName())
418 416
419 417 for parmElement in parmElementList:
420 418 parmConfObj = ParameterConf()
421 419 parmConfObj.readXml(parmElement)
422 420
423 421 #Compatible with old signal chain version
424 422 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
425 423 if self.type != 'self' and self.name == 'Plot':
426 424 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
427 425 self.name = parmConfObj.value
428 426 continue
429 427
430 428 self.parmConfObjList.append(parmConfObj)
431 429
432 430 def printattr(self):
433 431
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
435 433 self.id,
436 434 self.name,
437 435 self.type,
438 436 self.priority)
439 437
440 438 for parmConfObj in self.parmConfObjList:
441 439 parmConfObj.printattr()
442 440
443 441 def createObject(self, plotter_queue=None):
444 442
445 443
446 444 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
445 raise ValueError, 'This operation type cannot be created'
448 446
449 if self.type == 'plotter':
450 #Plotter(plotter_name)
447 if self.type == 'plotter':
451 448 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453 450
454 451 opObj = Plotter(self.name, plotter_queue)
455 452
456 453 if self.type == 'external' or self.type == 'other':
457 454
458 455 className = eval(self.name)
459 456 kwargs = self.getKwargs()
460 457
461 458 opObj = className(**kwargs)
462 459
463 460 return opObj
464 461
465 462
466 463 class ProcUnitConf():
467 464
468 465 id = None
469 466 name = None
470 467 datatype = None
471 468 inputId = None
472 469 parentId = None
473 470
474 471 opConfObjList = []
475 472
476 473 procUnitObj = None
477 474 opObjList = []
478 475
479 476 ELEMENTNAME = 'ProcUnit'
480 477
481 478 def __init__(self):
482 479
483 480 self.id = None
484 481 self.datatype = None
485 482 self.name = None
486 483 self.inputId = None
487 484
488 485 self.opConfObjList = []
489 486
490 487 self.procUnitObj = None
491 488 self.opObjDict = {}
492 489
493 490 def __getPriority(self):
494 491
495 492 return len(self.opConfObjList)+1
496 493
497 494 def __getNewId(self):
498 495
499 496 return int(self.id)*10 + len(self.opConfObjList) + 1
500 497
501 498 def getElementName(self):
502 499
503 500 return self.ELEMENTNAME
504 501
505 502 def getId(self):
506 503
507 504 return self.id
508 505
509 506 def updateId(self, new_id, parentId=parentId):
510 507
511 508
512 509 new_id = int(parentId)*10 + (int(self.id) % 10)
513 510 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
514 511
515 512 #If this proc unit has not inputs
516 513 if self.inputId == '0':
517 514 new_inputId = 0
518 515
519 516 n = 1
520 517 for opConfObj in self.opConfObjList:
521 518
522 519 idOp = str(int(new_id)*10 + n)
523 520 opConfObj.updateId(idOp)
524 521
525 522 n += 1
526 523
527 524 self.parentId = str(parentId)
528 525 self.id = str(new_id)
529 526 self.inputId = str(new_inputId)
530 527
531 528
532 529 def getInputId(self):
533 530
534 531 return self.inputId
535 532
536 533 def getOperationObjList(self):
537 534
538 535 return self.opConfObjList
539 536
540 537 def getOperationObj(self, name=None):
541 538
542 539 for opConfObj in self.opConfObjList:
543 540
544 541 if opConfObj.name != name:
545 542 continue
546 543
547 544 return opConfObj
548 545
549 546 return None
550 547
551 548 def getOpObjfromParamValue(self, value=None):
552 549
553 550 for opConfObj in self.opConfObjList:
554 551 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
555 552 continue
556 553 return opConfObj
557 554 return None
558 555
559 556 def getProcUnitObj(self):
560 557
561 558 return self.procUnitObj
562 559
563 560 def setup(self, id, name, datatype, inputId, parentId=None):
564 561
565 562 #Compatible with old signal chain version
566 563 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
564 raise ValueError, 'datatype or name should be defined'
568 565
569 566 if name==None:
570 567 if 'Proc' in datatype:
571 568 name = datatype
572 569 else:
573 570 name = '%sProc' %(datatype)
574 571
575 572 if datatype==None:
576 573 datatype = name.replace('Proc','')
577 574
578 575 self.id = str(id)
579 576 self.name = name
580 577 self.datatype = datatype
581 578 self.inputId = inputId
582 579 self.parentId = parentId
583 580
584 581 self.opConfObjList = []
585 582
586 583 self.addOperation(name='run', optype='self')
587 584
588 585 def removeOperations(self):
589 586
590 587 for obj in self.opConfObjList:
591 588 del obj
592 589
593 590 self.opConfObjList = []
594 591 self.addOperation(name='run')
595 592
596 593 def addParameter(self, **kwargs):
597 594 '''
598 Add parameters to "run" operation
595 Add parameters to 'run' operation
599 596 '''
600 597 opObj = self.opConfObjList[0]
601 598
602 599 opObj.addParameter(**kwargs)
603 600
604 601 return opObj
605 602
606 603 def addOperation(self, name, optype='self'):
607 604
608 605 id = self.__getNewId()
609 606 priority = self.__getPriority()
610 607
611 608 opConfObj = OperationConf()
612 609 opConfObj.setup(id, name=name, priority=priority, type=optype)
613 610
614 611 self.opConfObjList.append(opConfObj)
615 612
616 613 return opConfObj
617 614
618 615 def makeXml(self, projectElement):
619 616
620 617 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
621 618 procUnitElement.set('id', str(self.id))
622 619 procUnitElement.set('name', self.name)
623 620 procUnitElement.set('datatype', self.datatype)
624 621 procUnitElement.set('inputId', str(self.inputId))
625 622
626 623 for opConfObj in self.opConfObjList:
627 624 opConfObj.makeXml(procUnitElement)
628 625
629 626 def readXml(self, upElement):
630 627
631 628 self.id = upElement.get('id')
632 629 self.name = upElement.get('name')
633 630 self.datatype = upElement.get('datatype')
634 631 self.inputId = upElement.get('inputId')
635 632
636 if self.ELEMENTNAME == "ReadUnit":
637 self.datatype = self.datatype.replace("Reader", "")
633 if self.ELEMENTNAME == 'ReadUnit':
634 self.datatype = self.datatype.replace('Reader', '')
638 635
639 if self.ELEMENTNAME == "ProcUnit":
640 self.datatype = self.datatype.replace("Proc", "")
636 if self.ELEMENTNAME == 'ProcUnit':
637 self.datatype = self.datatype.replace('Proc', '')
641 638
642 639 if self.inputId == 'None':
643 640 self.inputId = '0'
644 641
645 642 self.opConfObjList = []
646 643
647 644 opElementList = upElement.iter(OperationConf().getElementName())
648 645
649 646 for opElement in opElementList:
650 647 opConfObj = OperationConf()
651 648 opConfObj.readXml(opElement)
652 649 self.opConfObjList.append(opConfObj)
653 650
654 651 def printattr(self):
655 652
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
657 654 self.id,
658 655 self.name,
659 656 self.datatype,
660 657 self.inputId)
661 658
662 659 for opConfObj in self.opConfObjList:
663 660 opConfObj.printattr()
664 661
665 662
666 663 def getKwargs(self):
667 664
668 665 opObj = self.opConfObjList[0]
669 666 kwargs = opObj.getKwargs()
670 667
671 668 return kwargs
672 669
673 670 def createObjects(self, plotter_queue=None):
674 671
675 672 className = eval(self.name)
676 673 kwargs = self.getKwargs()
677 674 procUnitObj = className(**kwargs)
678 675
679 676 for opConfObj in self.opConfObjList:
680 677
681 678 if opConfObj.type=='self' and self.name=='run':
682 679 continue
683 680 elif opConfObj.type=='self':
684 681 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
685 682 continue
686 683
687 684 opObj = opConfObj.createObject(plotter_queue)
688 685
689 686 self.opObjDict[opConfObj.id] = opObj
690 687
691 688 procUnitObj.addOperation(opObj, opConfObj.id)
692 689
693 690 self.procUnitObj = procUnitObj
694 691
695 692 return procUnitObj
696 693
697 694 def run(self):
698 695
699 696 is_ok = False
700 697
701 698 for opConfObj in self.opConfObjList:
702 699
703 700 kwargs = {}
704 701 for parmConfObj in opConfObj.getParameterObjList():
705 702 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
706 703 continue
707 704
708 705 kwargs[parmConfObj.name] = parmConfObj.getValue()
709 706
710 #ini = time.time()
711
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
713 707 sts = self.procUnitObj.call(opType = opConfObj.type,
714 708 opName = opConfObj.name,
715 709 opId = opConfObj.id)
716
717 # total_time = time.time() - ini
718 #
719 # if total_time > 0.002:
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
721 710
722 711 is_ok = is_ok or sts
723 712
724 713 return is_ok
725 714
726 715 def close(self):
727 716
728 717 for opConfObj in self.opConfObjList:
729 718 if opConfObj.type == 'self':
730 719 continue
731 720
732 721 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
733 722 opObj.close()
734 723
735 724 self.procUnitObj.close()
736 725
737 726 return
738 727
739 728 class ReadUnitConf(ProcUnitConf):
740 729
741 730 path = None
742 731 startDate = None
743 732 endDate = None
744 733 startTime = None
745 734 endTime = None
746 735
747 736 ELEMENTNAME = 'ReadUnit'
748 737
749 738 def __init__(self):
750 739
751 740 self.id = None
752 741 self.datatype = None
753 742 self.name = None
754 743 self.inputId = None
755 744
756 745 self.parentId = None
757 746
758 747 self.opConfObjList = []
759 748 self.opObjList = []
760 749
761 750 def getElementName(self):
762 751
763 752 return self.ELEMENTNAME
764 753
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
767 757 #Compatible with old signal chain version
768 758 if datatype==None and name==None:
769 raise ValueError, "datatype or name should be defined"
759 raise ValueError, 'datatype or name should be defined'
770 760
771 761 if name==None:
772 762 if 'Reader' in datatype:
773 763 name = datatype
774 764 else:
775 765 name = '%sReader' %(datatype)
776 766 if datatype==None:
777 767 datatype = name.replace('Reader','')
778 768
779 769 self.id = id
780 770 self.name = name
781 771 self.datatype = datatype
782 772 if path != '':
783 773 self.path = os.path.abspath(path)
784 774 self.startDate = startDate
785 775 self.endDate = endDate
786 776 self.startTime = startTime
787 777 self.endTime = endTime
788
789 778 self.inputId = '0'
790 779 self.parentId = parentId
791 self.queue = queue
792 780 self.server = server
793 781 self.addRunOperation(**kwargs)
794 782
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
783 def update(self, **kwargs):
796 784
797 #Compatible with old signal chain version
798 if datatype==None and name==None:
799 raise ValueError, "datatype or name should be defined"
800
801 if name==None:
785 if 'datatype' in kwargs:
786 datatype = kwargs.pop('datatype')
802 787 if 'Reader' in datatype:
803 name = datatype
788 self.name = datatype
804 789 else:
805 name = '%sReader' %(datatype)
806
807 if datatype==None:
808 datatype = name.replace('Reader','')
809
810 self.datatype = datatype
811 self.name = name
812 self.path = path
813 self.startDate = startDate
814 self.endDate = endDate
815 self.startTime = startTime
816 self.endTime = endTime
790 self.name = '%sReader' %(datatype)
791 self.datatype = self.name.replace('Reader', '')
817 792
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
794
795 for attr in attrs:
796 if attr in kwargs:
797 setattr(self, attr, kwargs.pop(attr))
798
818 799 self.inputId = '0'
819 self.parentId = parentId
820
821 800 self.updateRunOperation(**kwargs)
822 801
823 802 def removeOperations(self):
824 803
825 804 for obj in self.opConfObjList:
826 805 del obj
827 806
828 807 self.opConfObjList = []
829 808
830 809 def addRunOperation(self, **kwargs):
831 810
832 811 opObj = self.addOperation(name = 'run', optype = 'self')
833 812
834 813 if self.server is None:
835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
836 opObj.addParameter(name='path' , value=self.path, format='str')
837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
815 opObj.addParameter(name='path', value=self.path, format='str')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
820
842 821 for key, value in kwargs.items():
843 822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
844 823 else:
845 824 opObj.addParameter(name='server' , value=self.server, format='str')
846 825
847 826
848 827 return opObj
849 828
850 829 def updateRunOperation(self, **kwargs):
851 830
852 opObj = self.getOperationObj(name = 'run')
831 opObj = self.getOperationObj(name='run')
853 832 opObj.removeParameters()
854 833
855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
856 opObj.addParameter(name='path' , value=self.path, format='str')
857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
861
834 opObj.addParameter(name='datatype', value=self.datatype, format='str')
835 opObj.addParameter(name='path', value=self.path, format='str')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
837 opObj.addParameter(name='endDate', value=self.endDate, format='date')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
839 opObj.addParameter(name='endTime', value=self.endTime, format='time')
840
862 841 for key, value in kwargs.items():
863 842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
864 843
865 844 return opObj
866 845
867 # def makeXml(self, projectElement):
868 #
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
870 # procUnitElement.set('id', str(self.id))
871 # procUnitElement.set('name', self.name)
872 # procUnitElement.set('datatype', self.datatype)
873 # procUnitElement.set('inputId', str(self.inputId))
874 #
875 # for opConfObj in self.opConfObjList:
876 # opConfObj.makeXml(procUnitElement)
877
878 846 def readXml(self, upElement):
879 847
880 848 self.id = upElement.get('id')
881 849 self.name = upElement.get('name')
882 850 self.datatype = upElement.get('datatype')
883 851 self.inputId = upElement.get('inputId')
884 852
885 if self.ELEMENTNAME == "ReadUnit":
886 self.datatype = self.datatype.replace("Reader", "")
853 if self.ELEMENTNAME == 'ReadUnit':
854 self.datatype = self.datatype.replace('Reader', '')
887 855
888 856 if self.inputId == 'None':
889 857 self.inputId = '0'
890 858
891 859 self.opConfObjList = []
892 860
893 861 opElementList = upElement.iter(OperationConf().getElementName())
894 862
895 863 for opElement in opElementList:
896 864 opConfObj = OperationConf()
897 865 opConfObj.readXml(opElement)
898 866 self.opConfObjList.append(opConfObj)
899 867
900 868 if opConfObj.name == 'run':
901 869 self.path = opConfObj.getParameterValue('path')
902 870 self.startDate = opConfObj.getParameterValue('startDate')
903 871 self.endDate = opConfObj.getParameterValue('endDate')
904 872 self.startTime = opConfObj.getParameterValue('startTime')
905 873 self.endTime = opConfObj.getParameterValue('endTime')
906 874
907 875 class Project(Process):
876
908 877 id = None
909 name = None
878 # name = None
910 879 description = None
911 880 filename = None
912 881
913 882 procUnitConfObjDict = None
914 883
915 884 ELEMENTNAME = 'Project'
916 885
917 886 plotterQueue = None
918 887
919 def __init__(self, plotter_queue=None, logfile=None):
888 def __init__(self, plotter_queue=None):
889
920 890 Process.__init__(self)
921 891 self.id = None
922 self.name = None
892 # self.name = None
923 893 self.description = None
924 if logfile is not None:
925 logToFile(logfile)
894
926 895 self.plotterQueue = plotter_queue
927 896
928 897 self.procUnitConfObjDict = {}
929
898
930 899 def __getNewId(self):
931 900
932 901 idList = self.procUnitConfObjDict.keys()
933 902
934 903 id = int(self.id)*10
935 904
936 905 while True:
937 906 id += 1
938 907
939 908 if str(id) in idList:
940 909 continue
941 910
942 911 break
943 912
944 913 return str(id)
945 914
946 915 def getElementName(self):
947 916
948 917 return self.ELEMENTNAME
949 918
950 919 def getId(self):
951 920
952 921 return self.id
953 922
954 923 def updateId(self, new_id):
955 924
956 925 self.id = str(new_id)
957 926
958 927 keyList = self.procUnitConfObjDict.keys()
959 928 keyList.sort()
960 929
961 930 n = 1
962 931 newProcUnitConfObjDict = {}
963 932
964 933 for procKey in keyList:
965 934
966 935 procUnitConfObj = self.procUnitConfObjDict[procKey]
967 936 idProcUnit = str(int(self.id)*10 + n)
968 937 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
969 938
970 939 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
971 940 n += 1
972 941
973 942 self.procUnitConfObjDict = newProcUnitConfObjDict
974 943
975 def setup(self, id, name, description):
944 def setup(self, id, name='', description=''):
976 945
946 print
947 print '*'*60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
950 print
977 951 self.id = str(id)
978 self.name = name
979 952 self.description = description
980 953
981 954 def update(self, name, description):
982 955
983 self.name = name
984 956 self.description = description
985 957
958 def clone(self):
959
960 p = Project()
961 p.procUnitConfObjDict = self.procUnitConfObjDict
962 return p
963
986 964 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965
987 966 if id is None:
988 967 idReadUnit = self.__getNewId()
989 968 else:
990 969 idReadUnit = str(id)
991 970
992 971 readUnitConfObj = ReadUnitConf()
993 972 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
994 973
995 974 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
996 975
997 976 return readUnitConfObj
998 977
999 978 def addProcUnit(self, inputId='0', datatype=None, name=None):
1000 979
1001 980 idProcUnit = self.__getNewId()
1002 981
1003 982 procUnitConfObj = ProcUnitConf()
1004 983 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
1005 984
1006 985 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1007 986
1008 987 return procUnitConfObj
1009 988
1010 989 def removeProcUnit(self, id):
1011 990
1012 991 if id in self.procUnitConfObjDict.keys():
1013 992 self.procUnitConfObjDict.pop(id)
1014 993
1015 994 def getReadUnitId(self):
1016 995
1017 996 readUnitConfObj = self.getReadUnitObj()
1018 997
1019 998 return readUnitConfObj.id
1020 999
1021 1000 def getReadUnitObj(self):
1022 1001
1023 1002 for obj in self.procUnitConfObjDict.values():
1024 if obj.getElementName() == "ReadUnit":
1003 if obj.getElementName() == 'ReadUnit':
1025 1004 return obj
1026 1005
1027 1006 return None
1028 1007
1029 1008 def getProcUnitObj(self, id=None, name=None):
1030 1009
1031 1010 if id != None:
1032 1011 return self.procUnitConfObjDict[id]
1033 1012
1034 1013 if name != None:
1035 1014 return self.getProcUnitObjByName(name)
1036 1015
1037 1016 return None
1038 1017
1039 1018 def getProcUnitObjByName(self, name):
1040 1019
1041 1020 for obj in self.procUnitConfObjDict.values():
1042 1021 if obj.name == name:
1043 1022 return obj
1044 1023
1045 1024 return None
1046 1025
1047 1026 def procUnitItems(self):
1048 1027
1049 1028 return self.procUnitConfObjDict.items()
1050 1029
1051 1030 def makeXml(self):
1052 1031
1053 1032 projectElement = Element('Project')
1054 1033 projectElement.set('id', str(self.id))
1055 1034 projectElement.set('name', self.name)
1056 1035 projectElement.set('description', self.description)
1057 1036
1058 1037 for procUnitConfObj in self.procUnitConfObjDict.values():
1059 1038 procUnitConfObj.makeXml(projectElement)
1060 1039
1061 1040 self.projectElement = projectElement
1062 1041
1063 1042 def writeXml(self, filename=None):
1064 1043
1065 1044 if filename == None:
1066 1045 if self.filename:
1067 1046 filename = self.filename
1068 1047 else:
1069 filename = "schain.xml"
1048 filename = 'schain.xml'
1070 1049
1071 1050 if not filename:
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1051 print 'filename has not been defined. Use setFilename(filename) for do it.'
1073 1052 return 0
1074 1053
1075 1054 abs_file = os.path.abspath(filename)
1076 1055
1077 1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1079 1058 return 0
1080 1059
1081 1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1082 print "File %s already exists and it could not be overwriten" %abs_file
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1083 1062 return 0
1084 1063
1085 1064 self.makeXml()
1086 1065
1087 1066 ElementTree(self.projectElement).write(abs_file, method='xml')
1088 1067
1089 1068 self.filename = abs_file
1090 1069
1091 1070 return 1
1092 1071
1093 1072 def readXml(self, filename = None):
1094 1073
1095 1074 if not filename:
1096 print "filename is not defined"
1075 print 'filename is not defined'
1097 1076 return 0
1098 1077
1099 1078 abs_file = os.path.abspath(filename)
1100 1079
1101 1080 if not os.path.isfile(abs_file):
1102 print "%s file does not exist" %abs_file
1081 print '%s file does not exist' %abs_file
1103 1082 return 0
1104 1083
1105 1084 self.projectElement = None
1106 1085 self.procUnitConfObjDict = {}
1107 1086
1108 1087 try:
1109 1088 self.projectElement = ElementTree().parse(abs_file)
1110 1089 except:
1111 print "Error reading %s, verify file format" %filename
1090 print 'Error reading %s, verify file format' %filename
1112 1091 return 0
1113 1092
1114 1093 self.project = self.projectElement.tag
1115 1094
1116 1095 self.id = self.projectElement.get('id')
1117 1096 self.name = self.projectElement.get('name')
1118 1097 self.description = self.projectElement.get('description')
1119 1098
1120 1099 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1121 1100
1122 1101 for readUnitElement in readUnitElementList:
1123 1102 readUnitConfObj = ReadUnitConf()
1124 1103 readUnitConfObj.readXml(readUnitElement)
1125 1104
1126 1105 if readUnitConfObj.parentId == None:
1127 1106 readUnitConfObj.parentId = self.id
1128 1107
1129 1108 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1130 1109
1131 1110 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1132 1111
1133 1112 for procUnitElement in procUnitElementList:
1134 1113 procUnitConfObj = ProcUnitConf()
1135 1114 procUnitConfObj.readXml(procUnitElement)
1136 1115
1137 1116 if procUnitConfObj.parentId == None:
1138 1117 procUnitConfObj.parentId = self.id
1139 1118
1140 1119 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1141 1120
1142 1121 self.filename = abs_file
1143 1122
1144 1123 return 1
1145 1124
1146 1125 def printattr(self):
1147 1126
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1149 self.name,
1150 self.description)
1151
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1128 self.name,
1129 self.description)
1130
1152 1131 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 1132 procUnitConfObj.printattr()
1154 1133
1155 1134 def createObjects(self):
1156 1135
1157 1136 for procUnitConfObj in self.procUnitConfObjDict.values():
1158 1137 procUnitConfObj.createObjects(self.plotterQueue)
1159 1138
1160 1139 def __connect(self, objIN, thisObj):
1161 1140
1162 1141 thisObj.setInput(objIN.getOutputObj())
1163 1142
1164 1143 def connectObjects(self):
1165 1144
1166 1145 for thisPUConfObj in self.procUnitConfObjDict.values():
1167 1146
1168 1147 inputId = thisPUConfObj.getInputId()
1169 1148
1170 1149 if int(inputId) == 0:
1171 1150 continue
1172 1151
1173 1152 #Get input object
1174 1153 puConfINObj = self.procUnitConfObjDict[inputId]
1175 1154 puObjIN = puConfINObj.getProcUnitObj()
1176 1155
1177 1156 #Get current object
1178 1157 thisPUObj = thisPUConfObj.getProcUnitObj()
1179 1158
1180 1159 self.__connect(puObjIN, thisPUObj)
1181 1160
1182 def __handleError(self, procUnitConfObj, send_email=True):
1161 def __handleError(self, procUnitConfObj, send_email=False):
1183 1162
1184 1163 import socket
1185 1164
1186 1165 err = traceback.format_exception(sys.exc_info()[0],
1187 1166 sys.exc_info()[1],
1188 1167 sys.exc_info()[2])
1189 1168
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1191 print "***** %s" %err[-1]
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1170 print '***** %s' %err[-1]
1192 1171
1193 message = "".join(err)
1172 message = ''.join(err)
1194 1173
1195 1174 sys.stderr.write(message)
1196 1175
1197 1176 if not send_email:
1198 1177 return
1199 1178
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1201 1180
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1205 subtitle += "Configuration file: %s\n" %self.filename
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1184 subtitle += 'Configuration file: %s\n' %self.filename
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1207 1186
1208 1187 readUnitConfObj = self.getReadUnitObj()
1209 1188 if readUnitConfObj:
1210 subtitle += "\nInput parameters:\n"
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1189 subtitle += '\nInput parameters:\n'
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1217 1196
1218 1197 adminObj = schainpy.admin.SchainNotify()
1219 1198 adminObj.sendAlert(message=message,
1220 1199 subject=subject,
1221 1200 subtitle=subtitle,
1222 1201 filename=self.filename)
1223 1202
1224 1203 def isPaused(self):
1225 1204 return 0
1226 1205
1227 1206 def isStopped(self):
1228 1207 return 0
1229 1208
1230 1209 def runController(self):
1231 """
1210 '''
1232 1211 returns 0 when this process has been stopped, 1 otherwise
1233 """
1212 '''
1234 1213
1235 1214 if self.isPaused():
1236 print "Process suspended"
1215 print 'Process suspended'
1237 1216
1238 1217 while True:
1239 sleep(0.1)
1218 time.sleep(0.1)
1240 1219
1241 1220 if not self.isPaused():
1242 1221 break
1243 1222
1244 1223 if self.isStopped():
1245 1224 break
1246 1225
1247 print "Process reinitialized"
1226 print 'Process reinitialized'
1248 1227
1249 1228 if self.isStopped():
1250 print "Process stopped"
1229 print 'Process stopped'
1251 1230 return 0
1252 1231
1253 1232 return 1
1254 1233
1255 1234 def setFilename(self, filename):
1256 1235
1257 1236 self.filename = filename
1258 1237
1259 1238 def setPlotterQueue(self, plotter_queue):
1260 1239
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1240 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1262 1241
1263 1242 def getPlotterQueue(self):
1264 1243
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1244 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1266 1245
1267 1246 def useExternalPlotter(self):
1268 1247
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1248 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270 1249
1250 def run(self):
1271 1251
1272 def run(self, filename=None):
1252 log.success('Starting {}'.format(self.name))
1273 1253
1274 # self.writeXml(filename)
1275 1254 self.createObjects()
1276 1255 self.connectObjects()
1277 1256
1278 print
1279 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1282 print
1283
1284 1257 keyList = self.procUnitConfObjDict.keys()
1285 1258 keyList.sort()
1286 1259
1287 1260 while(True):
1288 1261
1289 1262 is_ok = False
1290 1263
1291 1264 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293 1265
1294 1266 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 1267
1296 1268 try:
1297 1269 sts = procUnitConfObj.run()
1298 1270 is_ok = is_ok or sts
1299 1271 except KeyboardInterrupt:
1300 1272 is_ok = False
1301 1273 break
1302 1274 except ValueError, e:
1303 sleep(0.5)
1275 time.sleep(0.5)
1304 1276 self.__handleError(procUnitConfObj, send_email=True)
1305 1277 is_ok = False
1306 1278 break
1307 1279 except:
1308 sleep(0.5)
1280 time.sleep(0.5)
1309 1281 self.__handleError(procUnitConfObj)
1310 1282 is_ok = False
1311 1283 break
1312 1284
1313 1285 #If every process unit finished so end process
1314 1286 if not(is_ok):
1315 # print "Every process unit have finished"
1316 1287 break
1317 1288
1318 1289 if not self.runController():
1319 1290 break
1320 1291
1321 1292 #Closing every process
1322 1293 for procKey in keyList:
1323 1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 1295 procUnitConfObj.close()
1296
1297 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,1218 +1,1218
1 1 '''
2 2
3 3 $Author: murco $
4 4 $Id: JROData.py 173 2012-11-20 15:06:21Z murco $
5 5 '''
6 6
7 7 import copy
8 8 import numpy
9 9 import datetime
10 10
11 11 from jroheaderIO import SystemHeader, RadarControllerHeader
12 12 from schainpy import cSchain
13 13
14 14
15 15 def getNumpyDtype(dataTypeCode):
16 16
17 17 if dataTypeCode == 0:
18 18 numpyDtype = numpy.dtype([('real','<i1'),('imag','<i1')])
19 19 elif dataTypeCode == 1:
20 20 numpyDtype = numpy.dtype([('real','<i2'),('imag','<i2')])
21 21 elif dataTypeCode == 2:
22 22 numpyDtype = numpy.dtype([('real','<i4'),('imag','<i4')])
23 23 elif dataTypeCode == 3:
24 24 numpyDtype = numpy.dtype([('real','<i8'),('imag','<i8')])
25 25 elif dataTypeCode == 4:
26 26 numpyDtype = numpy.dtype([('real','<f4'),('imag','<f4')])
27 27 elif dataTypeCode == 5:
28 28 numpyDtype = numpy.dtype([('real','<f8'),('imag','<f8')])
29 29 else:
30 30 raise ValueError, 'dataTypeCode was not defined'
31 31
32 32 return numpyDtype
33 33
34 34 def getDataTypeCode(numpyDtype):
35 35
36 36 if numpyDtype == numpy.dtype([('real','<i1'),('imag','<i1')]):
37 37 datatype = 0
38 38 elif numpyDtype == numpy.dtype([('real','<i2'),('imag','<i2')]):
39 39 datatype = 1
40 40 elif numpyDtype == numpy.dtype([('real','<i4'),('imag','<i4')]):
41 41 datatype = 2
42 42 elif numpyDtype == numpy.dtype([('real','<i8'),('imag','<i8')]):
43 43 datatype = 3
44 44 elif numpyDtype == numpy.dtype([('real','<f4'),('imag','<f4')]):
45 45 datatype = 4
46 46 elif numpyDtype == numpy.dtype([('real','<f8'),('imag','<f8')]):
47 47 datatype = 5
48 48 else:
49 49 datatype = None
50 50
51 51 return datatype
52 52
53 53 def hildebrand_sekhon(data, navg):
54 54 """
55 55 This method is for the objective determination of the noise level in Doppler spectra. This
56 56 implementation technique is based on the fact that the standard deviation of the spectral
57 57 densities is equal to the mean spectral density for white Gaussian noise
58 58
59 59 Inputs:
60 60 Data : heights
61 61 navg : numbers of averages
62 62
63 63 Return:
64 64 -1 : any error
65 65 anoise : noise's level
66 66 """
67 67
68 68 sortdata = numpy.sort(data, axis=None)
69 69 # lenOfData = len(sortdata)
70 70 # nums_min = lenOfData*0.2
71 71 #
72 72 # if nums_min <= 5:
73 73 # nums_min = 5
74 74 #
75 75 # sump = 0.
76 76 #
77 77 # sumq = 0.
78 78 #
79 79 # j = 0
80 80 #
81 81 # cont = 1
82 82 #
83 83 # while((cont==1)and(j<lenOfData)):
84 84 #
85 85 # sump += sortdata[j]
86 86 #
87 87 # sumq += sortdata[j]**2
88 88 #
89 89 # if j > nums_min:
90 90 # rtest = float(j)/(j-1) + 1.0/navg
91 91 # if ((sumq*j) > (rtest*sump**2)):
92 92 # j = j - 1
93 93 # sump = sump - sortdata[j]
94 94 # sumq = sumq - sortdata[j]**2
95 95 # cont = 0
96 96 #
97 97 # j += 1
98 98 #
99 99 # lnoise = sump /j
100 100 #
101 101 # return lnoise
102 102
103 103 return cSchain.hildebrand_sekhon(sortdata, navg)
104 104
105 105
106 106 class Beam:
107 107
108 108 def __init__(self):
109 109 self.codeList = []
110 110 self.azimuthList = []
111 111 self.zenithList = []
112 112
113 113 class GenericData(object):
114 114
115 115 flagNoData = True
116 116
117 117 def copy(self, inputObj=None):
118 118
119 119 if inputObj == None:
120 120 return copy.deepcopy(self)
121 121
122 122 for key in inputObj.__dict__.keys():
123 123
124 124 attribute = inputObj.__dict__[key]
125 125
126 126 #If this attribute is a tuple or list
127 127 if type(inputObj.__dict__[key]) in (tuple, list):
128 128 self.__dict__[key] = attribute[:]
129 129 continue
130 130
131 131 #If this attribute is another object or instance
132 132 if hasattr(attribute, '__dict__'):
133 133 self.__dict__[key] = attribute.copy()
134 134 continue
135 135
136 136 self.__dict__[key] = inputObj.__dict__[key]
137 137
138 138 def deepcopy(self):
139 139
140 140 return copy.deepcopy(self)
141 141
142 142 def isEmpty(self):
143 143
144 144 return self.flagNoData
145 145
146 146 class JROData(GenericData):
147 147
148 148 # m_BasicHeader = BasicHeader()
149 149 # m_ProcessingHeader = ProcessingHeader()
150 150
151 151 systemHeaderObj = SystemHeader()
152 152
153 153 radarControllerHeaderObj = RadarControllerHeader()
154 154
155 155 # data = None
156 156
157 157 type = None
158 158
159 159 datatype = None #dtype but in string
160 160
161 161 # dtype = None
162 162
163 163 # nChannels = None
164 164
165 165 # nHeights = None
166 166
167 167 nProfiles = None
168 168
169 169 heightList = None
170 170
171 171 channelList = None
172 172
173 173 flagDiscontinuousBlock = False
174 174
175 175 useLocalTime = False
176 176
177 177 utctime = None
178 178
179 179 timeZone = None
180 180
181 181 dstFlag = None
182 182
183 183 errorCount = None
184 184
185 185 blocksize = None
186 186
187 187 # nCode = None
188 188 #
189 189 # nBaud = None
190 190 #
191 191 # code = None
192 192
193 193 flagDecodeData = False #asumo q la data no esta decodificada
194 194
195 195 flagDeflipData = False #asumo q la data no esta sin flip
196 196
197 197 flagShiftFFT = False
198 198
199 199 # ippSeconds = None
200 200
201 201 # timeInterval = None
202 202
203 203 nCohInt = None
204 204
205 205 # noise = None
206 206
207 207 windowOfFilter = 1
208 208
209 209 #Speed of ligth
210 210 C = 3e8
211 211
212 212 frequency = 49.92e6
213 213
214 214 realtime = False
215 215
216 216 beacon_heiIndexList = None
217 217
218 218 last_block = None
219 219
220 220 blocknow = None
221 221
222 222 azimuth = None
223 223
224 224 zenith = None
225 225
226 226 beam = Beam()
227 227
228 228 profileIndex = None
229 229
230 230 def getNoise(self):
231 231
232 232 raise NotImplementedError
233 233
234 234 def getNChannels(self):
235 235
236 236 return len(self.channelList)
237 237
238 238 def getChannelIndexList(self):
239 239
240 240 return range(self.nChannels)
241 241
242 242 def getNHeights(self):
243 243
244 244 return len(self.heightList)
245 245
246 246 def getHeiRange(self, extrapoints=0):
247 247
248 248 heis = self.heightList
249 249 # deltah = self.heightList[1] - self.heightList[0]
250 250 #
251 251 # heis.append(self.heightList[-1])
252 252
253 253 return heis
254 254
255 255 def getDeltaH(self):
256 256
257 257 delta = self.heightList[1] - self.heightList[0]
258 258
259 259 return delta
260 260
261 261 def getltctime(self):
262 262
263 263 if self.useLocalTime:
264 264 return self.utctime - self.timeZone*60
265 265
266 266 return self.utctime
267 267
268 268 def getDatatime(self):
269 269
270 270 datatimeValue = datetime.datetime.utcfromtimestamp(self.ltctime)
271 271 return datatimeValue
272 272
273 273 def getTimeRange(self):
274 274
275 275 datatime = []
276 276
277 277 datatime.append(self.ltctime)
278 278 datatime.append(self.ltctime + self.timeInterval+1)
279 279
280 280 datatime = numpy.array(datatime)
281 281
282 282 return datatime
283 283
284 284 def getFmaxTimeResponse(self):
285 285
286 286 period = (10**-6)*self.getDeltaH()/(0.15)
287 287
288 288 PRF = 1./(period * self.nCohInt)
289 289
290 290 fmax = PRF
291 291
292 292 return fmax
293 293
294 294 def getFmax(self):
295 295 PRF = 1./(self.ippSeconds * self.nCohInt)
296 296
297 297 fmax = PRF
298 298 return fmax
299 299
300 300 def getVmax(self):
301 301
302 302 _lambda = self.C/self.frequency
303 303
304 304 vmax = self.getFmax() * _lambda/2
305 305
306 306 return vmax
307 307
308 308 def get_ippSeconds(self):
309 309 '''
310 310 '''
311 311 return self.radarControllerHeaderObj.ippSeconds
312 312
313 313 def set_ippSeconds(self, ippSeconds):
314 314 '''
315 315 '''
316 316
317 317 self.radarControllerHeaderObj.ippSeconds = ippSeconds
318 318
319 319 return
320 320
321 321 def get_dtype(self):
322 322 '''
323 323 '''
324 324 return getNumpyDtype(self.datatype)
325 325
326 326 def set_dtype(self, numpyDtype):
327 327 '''
328 328 '''
329 329
330 330 self.datatype = getDataTypeCode(numpyDtype)
331 331
332 332 def get_code(self):
333 333 '''
334 334 '''
335 335 return self.radarControllerHeaderObj.code
336 336
337 337 def set_code(self, code):
338 338 '''
339 339 '''
340 340 self.radarControllerHeaderObj.code = code
341 341
342 342 return
343 343
344 344 def get_ncode(self):
345 345 '''
346 346 '''
347 347 return self.radarControllerHeaderObj.nCode
348 348
349 349 def set_ncode(self, nCode):
350 350 '''
351 351 '''
352 352 self.radarControllerHeaderObj.nCode = nCode
353 353
354 354 return
355 355
356 356 def get_nbaud(self):
357 357 '''
358 358 '''
359 359 return self.radarControllerHeaderObj.nBaud
360 360
361 361 def set_nbaud(self, nBaud):
362 362 '''
363 363 '''
364 364 self.radarControllerHeaderObj.nBaud = nBaud
365 365
366 366 return
367 367
368 368 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
369 369 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
370 370 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
371 371 #noise = property(getNoise, "I'm the 'nHeights' property.")
372 372 datatime = property(getDatatime, "I'm the 'datatime' property")
373 373 ltctime = property(getltctime, "I'm the 'ltctime' property")
374 374 ippSeconds = property(get_ippSeconds, set_ippSeconds)
375 375 dtype = property(get_dtype, set_dtype)
376 376 # timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
377 377 code = property(get_code, set_code)
378 378 nCode = property(get_ncode, set_ncode)
379 379 nBaud = property(get_nbaud, set_nbaud)
380 380
381 381 class Voltage(JROData):
382 382
383 383 #data es un numpy array de 2 dmensiones (canales, alturas)
384 384 data = None
385 385
386 386 def __init__(self):
387 387 '''
388 388 Constructor
389 389 '''
390 390
391 391 self.useLocalTime = True
392 392
393 393 self.radarControllerHeaderObj = RadarControllerHeader()
394 394
395 395 self.systemHeaderObj = SystemHeader()
396 396
397 397 self.type = "Voltage"
398 398
399 399 self.data = None
400 400
401 401 # self.dtype = None
402 402
403 403 # self.nChannels = 0
404 404
405 405 # self.nHeights = 0
406 406
407 407 self.nProfiles = None
408 408
409 409 self.heightList = None
410 410
411 411 self.channelList = None
412 412
413 413 # self.channelIndexList = None
414 414
415 415 self.flagNoData = True
416 416
417 417 self.flagDiscontinuousBlock = False
418 418
419 419 self.utctime = None
420 420
421 421 self.timeZone = None
422 422
423 423 self.dstFlag = None
424 424
425 425 self.errorCount = None
426 426
427 427 self.nCohInt = None
428 428
429 429 self.blocksize = None
430 430
431 431 self.flagDecodeData = False #asumo q la data no esta decodificada
432 432
433 433 self.flagDeflipData = False #asumo q la data no esta sin flip
434 434
435 435 self.flagShiftFFT = False
436 436
437 437 self.flagDataAsBlock = False #Asumo que la data es leida perfil a perfil
438 438
439 439 self.profileIndex = 0
440 440
441 441 def getNoisebyHildebrand(self, channel = None):
442 442 """
443 443 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
444 444
445 445 Return:
446 446 noiselevel
447 447 """
448 448
449 449 if channel != None:
450 450 data = self.data[channel]
451 451 nChannels = 1
452 452 else:
453 453 data = self.data
454 454 nChannels = self.nChannels
455 455
456 456 noise = numpy.zeros(nChannels)
457 457 power = data * numpy.conjugate(data)
458 458
459 459 for thisChannel in range(nChannels):
460 460 if nChannels == 1:
461 461 daux = power[:].real
462 462 else:
463 463 daux = power[thisChannel,:].real
464 464 noise[thisChannel] = hildebrand_sekhon(daux, self.nCohInt)
465 465
466 466 return noise
467 467
468 468 def getNoise(self, type = 1, channel = None):
469 469
470 470 if type == 1:
471 471 noise = self.getNoisebyHildebrand(channel)
472 472
473 473 return noise
474 474
475 475 def getPower(self, channel = None):
476 476
477 477 if channel != None:
478 478 data = self.data[channel]
479 479 else:
480 480 data = self.data
481 481
482 482 power = data * numpy.conjugate(data)
483 483 powerdB = 10*numpy.log10(power.real)
484 484 powerdB = numpy.squeeze(powerdB)
485 485
486 486 return powerdB
487 487
488 488 def getTimeInterval(self):
489 489
490 490 timeInterval = self.ippSeconds * self.nCohInt
491 491
492 492 return timeInterval
493 493
494 494 noise = property(getNoise, "I'm the 'nHeights' property.")
495 495 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
496 496
497 497 class Spectra(JROData):
498 498
499 499 #data spc es un numpy array de 2 dmensiones (canales, perfiles, alturas)
500 500 data_spc = None
501 501
502 502 #data cspc es un numpy array de 2 dmensiones (canales, pares, alturas)
503 503 data_cspc = None
504 504
505 505 #data dc es un numpy array de 2 dmensiones (canales, alturas)
506 506 data_dc = None
507 507
508 508 #data power
509 509 data_pwr = None
510 510
511 511 nFFTPoints = None
512 512
513 513 # nPairs = None
514 514
515 515 pairsList = None
516 516
517 517 nIncohInt = None
518 518
519 519 wavelength = None #Necesario para cacular el rango de velocidad desde la frecuencia
520 520
521 521 nCohInt = None #se requiere para determinar el valor de timeInterval
522 522
523 523 ippFactor = None
524 524
525 525 profileIndex = 0
526 526
527 527 plotting = "spectra"
528 528
529 529 def __init__(self):
530 530 '''
531 531 Constructor
532 532 '''
533 533
534 534 self.useLocalTime = True
535 535
536 536 self.radarControllerHeaderObj = RadarControllerHeader()
537 537
538 538 self.systemHeaderObj = SystemHeader()
539 539
540 540 self.type = "Spectra"
541 541
542 542 # self.data = None
543 543
544 544 # self.dtype = None
545 545
546 546 # self.nChannels = 0
547 547
548 548 # self.nHeights = 0
549 549
550 550 self.nProfiles = None
551 551
552 552 self.heightList = None
553 553
554 554 self.channelList = None
555 555
556 556 # self.channelIndexList = None
557 557
558 558 self.pairsList = None
559 559
560 560 self.flagNoData = True
561 561
562 562 self.flagDiscontinuousBlock = False
563 563
564 564 self.utctime = None
565 565
566 566 self.nCohInt = None
567 567
568 568 self.nIncohInt = None
569 569
570 570 self.blocksize = None
571 571
572 572 self.nFFTPoints = None
573 573
574 574 self.wavelength = None
575 575
576 576 self.flagDecodeData = False #asumo q la data no esta decodificada
577 577
578 578 self.flagDeflipData = False #asumo q la data no esta sin flip
579 579
580 580 self.flagShiftFFT = False
581 581
582 582 self.ippFactor = 1
583 583
584 584 #self.noise = None
585 585
586 586 self.beacon_heiIndexList = []
587 587
588 588 self.noise_estimation = None
589 589
590 590
591 591 def getNoisebyHildebrand(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
592 592 """
593 593 Determino el nivel de ruido usando el metodo Hildebrand-Sekhon
594 594
595 595 Return:
596 596 noiselevel
597 597 """
598 598
599 599 noise = numpy.zeros(self.nChannels)
600 600
601 601 for channel in range(self.nChannels):
602 602 daux = self.data_spc[channel,xmin_index:xmax_index,ymin_index:ymax_index]
603 603 noise[channel] = hildebrand_sekhon(daux, self.nIncohInt)
604 604
605 605 return noise
606 606
607 607 def getNoise(self, xmin_index=None, xmax_index=None, ymin_index=None, ymax_index=None):
608 608
609 609 if self.noise_estimation is not None:
610 610 return self.noise_estimation #this was estimated by getNoise Operation defined in jroproc_spectra.py
611 611 else:
612 612 noise = self.getNoisebyHildebrand(xmin_index, xmax_index, ymin_index, ymax_index)
613 613 return noise
614 614
615 615 def getFreqRangeTimeResponse(self, extrapoints=0):
616 616
617 617 deltafreq = self.getFmaxTimeResponse() / (self.nFFTPoints*self.ippFactor)
618 618 freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
619 619
620 620 return freqrange
621 621
622 622 def getAcfRange(self, extrapoints=0):
623 623
624 624 deltafreq = 10./(self.getFmax() / (self.nFFTPoints*self.ippFactor))
625 625 freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
626 626
627 627 return freqrange
628 628
629 629 def getFreqRange(self, extrapoints=0):
630 630
631 631 deltafreq = self.getFmax() / (self.nFFTPoints*self.ippFactor)
632 632 freqrange = deltafreq*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) - deltafreq/2
633 633
634 634 return freqrange
635 635
636 636 def getVelRange(self, extrapoints=0):
637 637
638 638 deltav = self.getVmax() / (self.nFFTPoints*self.ippFactor)
639 639 velrange = deltav*(numpy.arange(self.nFFTPoints+extrapoints)-self.nFFTPoints/2.) #- deltav/2
640 640
641 641 return velrange
642 642
643 643 def getNPairs(self):
644 644
645 645 return len(self.pairsList)
646 646
647 647 def getPairsIndexList(self):
648 648
649 649 return range(self.nPairs)
650 650
651 651 def getNormFactor(self):
652 652
653 653 pwcode = 1
654 654
655 655 if self.flagDecodeData:
656 656 pwcode = numpy.sum(self.code[0]**2)
657 657 #normFactor = min(self.nFFTPoints,self.nProfiles)*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
658 658 normFactor = self.nProfiles*self.nIncohInt*self.nCohInt*pwcode*self.windowOfFilter
659 659
660 660 return normFactor
661 661
662 662 def getFlagCspc(self):
663 663
664 664 if self.data_cspc is None:
665 665 return True
666 666
667 667 return False
668 668
669 669 def getFlagDc(self):
670 670
671 671 if self.data_dc is None:
672 672 return True
673 673
674 674 return False
675 675
676 676 def getTimeInterval(self):
677 677
678 678 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt * self.nProfiles
679 679
680 680 return timeInterval
681 681
682 682 def getPower(self):
683 683
684 684 factor = self.normFactor
685 685 z = self.data_spc/factor
686 686 z = numpy.where(numpy.isfinite(z), z, numpy.NAN)
687 687 avg = numpy.average(z, axis=1)
688 688
689 689 return 10*numpy.log10(avg)
690 690
691 691 def getCoherence(self, pairsList=None, phase=False):
692 692
693 693 z = []
694 694 if pairsList is None:
695 695 pairsIndexList = self.pairsIndexList
696 696 else:
697 697 pairsIndexList = []
698 698 for pair in pairsList:
699 699 if pair not in self.pairsList:
700 700 raise ValueError, "Pair %s is not in dataOut.pairsList" %(pair)
701 pairsIndexList.append(self.pairsList.index(pair))
701 pairsIndexList.append(self.pairsList.index(pair))
702 702 for i in range(len(pairsIndexList)):
703 703 pair = self.pairsList[pairsIndexList[i]]
704 704 ccf = numpy.average(self.data_cspc[pairsIndexList[i], :, :], axis=0)
705 705 powa = numpy.average(self.data_spc[pair[0], :, :], axis=0)
706 706 powb = numpy.average(self.data_spc[pair[1], :, :], axis=0)
707 707 avgcoherenceComplex = ccf/numpy.sqrt(powa*powb)
708 708 if phase:
709 709 data = numpy.arctan2(avgcoherenceComplex.imag,
710 710 avgcoherenceComplex.real)*180/numpy.pi
711 711 else:
712 712 data = numpy.abs(avgcoherenceComplex)
713 713
714 714 z.append(data)
715 715
716 716 return numpy.array(z)
717 717
718 718 def setValue(self, value):
719 719
720 720 print "This property should not be initialized"
721 721
722 722 return
723 723
724 724 nPairs = property(getNPairs, setValue, "I'm the 'nPairs' property.")
725 725 pairsIndexList = property(getPairsIndexList, setValue, "I'm the 'pairsIndexList' property.")
726 726 normFactor = property(getNormFactor, setValue, "I'm the 'getNormFactor' property.")
727 727 flag_cspc = property(getFlagCspc, setValue)
728 728 flag_dc = property(getFlagDc, setValue)
729 729 noise = property(getNoise, setValue, "I'm the 'nHeights' property.")
730 730 timeInterval = property(getTimeInterval, setValue, "I'm the 'timeInterval' property")
731 731
732 732 class SpectraHeis(Spectra):
733 733
734 734 data_spc = None
735 735
736 736 data_cspc = None
737 737
738 738 data_dc = None
739 739
740 740 nFFTPoints = None
741 741
742 742 # nPairs = None
743 743
744 744 pairsList = None
745 745
746 746 nCohInt = None
747 747
748 748 nIncohInt = None
749 749
750 750 def __init__(self):
751 751
752 752 self.radarControllerHeaderObj = RadarControllerHeader()
753 753
754 754 self.systemHeaderObj = SystemHeader()
755 755
756 756 self.type = "SpectraHeis"
757 757
758 758 # self.dtype = None
759 759
760 760 # self.nChannels = 0
761 761
762 762 # self.nHeights = 0
763 763
764 764 self.nProfiles = None
765 765
766 766 self.heightList = None
767 767
768 768 self.channelList = None
769 769
770 770 # self.channelIndexList = None
771 771
772 772 self.flagNoData = True
773 773
774 774 self.flagDiscontinuousBlock = False
775 775
776 776 # self.nPairs = 0
777 777
778 778 self.utctime = None
779 779
780 780 self.blocksize = None
781 781
782 782 self.profileIndex = 0
783 783
784 784 self.nCohInt = 1
785 785
786 786 self.nIncohInt = 1
787 787
788 788 def getNormFactor(self):
789 789 pwcode = 1
790 790 if self.flagDecodeData:
791 791 pwcode = numpy.sum(self.code[0]**2)
792 792
793 793 normFactor = self.nIncohInt*self.nCohInt*pwcode
794 794
795 795 return normFactor
796 796
797 797 def getTimeInterval(self):
798 798
799 799 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
800 800
801 801 return timeInterval
802 802
803 803 normFactor = property(getNormFactor, "I'm the 'getNormFactor' property.")
804 804 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
805 805
806 806 class Fits(JROData):
807 807
808 808 heightList = None
809 809
810 810 channelList = None
811 811
812 812 flagNoData = True
813 813
814 814 flagDiscontinuousBlock = False
815 815
816 816 useLocalTime = False
817 817
818 818 utctime = None
819 819
820 820 timeZone = None
821 821
822 822 # ippSeconds = None
823 823
824 824 # timeInterval = None
825 825
826 826 nCohInt = None
827 827
828 828 nIncohInt = None
829 829
830 830 noise = None
831 831
832 832 windowOfFilter = 1
833 833
834 834 #Speed of ligth
835 835 C = 3e8
836 836
837 837 frequency = 49.92e6
838 838
839 839 realtime = False
840 840
841 841
842 842 def __init__(self):
843 843
844 844 self.type = "Fits"
845 845
846 846 self.nProfiles = None
847 847
848 848 self.heightList = None
849 849
850 850 self.channelList = None
851 851
852 852 # self.channelIndexList = None
853 853
854 854 self.flagNoData = True
855 855
856 856 self.utctime = None
857 857
858 858 self.nCohInt = 1
859 859
860 860 self.nIncohInt = 1
861 861
862 862 self.useLocalTime = True
863 863
864 864 self.profileIndex = 0
865 865
866 866 # self.utctime = None
867 867 # self.timeZone = None
868 868 # self.ltctime = None
869 869 # self.timeInterval = None
870 870 # self.header = None
871 871 # self.data_header = None
872 872 # self.data = None
873 873 # self.datatime = None
874 874 # self.flagNoData = False
875 875 # self.expName = ''
876 876 # self.nChannels = None
877 877 # self.nSamples = None
878 878 # self.dataBlocksPerFile = None
879 879 # self.comments = ''
880 880 #
881 881
882 882
883 883 def getltctime(self):
884 884
885 885 if self.useLocalTime:
886 886 return self.utctime - self.timeZone*60
887 887
888 888 return self.utctime
889 889
890 890 def getDatatime(self):
891 891
892 892 datatime = datetime.datetime.utcfromtimestamp(self.ltctime)
893 893 return datatime
894 894
895 895 def getTimeRange(self):
896 896
897 897 datatime = []
898 898
899 899 datatime.append(self.ltctime)
900 900 datatime.append(self.ltctime + self.timeInterval)
901 901
902 902 datatime = numpy.array(datatime)
903 903
904 904 return datatime
905 905
906 906 def getHeiRange(self):
907 907
908 908 heis = self.heightList
909 909
910 910 return heis
911 911
912 912 def getNHeights(self):
913 913
914 914 return len(self.heightList)
915 915
916 916 def getNChannels(self):
917 917
918 918 return len(self.channelList)
919 919
920 920 def getChannelIndexList(self):
921 921
922 922 return range(self.nChannels)
923 923
924 924 def getNoise(self, type = 1):
925 925
926 926 #noise = numpy.zeros(self.nChannels)
927 927
928 928 if type == 1:
929 929 noise = self.getNoisebyHildebrand()
930 930
931 931 if type == 2:
932 932 noise = self.getNoisebySort()
933 933
934 934 if type == 3:
935 935 noise = self.getNoisebyWindow()
936 936
937 937 return noise
938 938
939 939 def getTimeInterval(self):
940 940
941 941 timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
942 942
943 943 return timeInterval
944 944
945 945 datatime = property(getDatatime, "I'm the 'datatime' property")
946 946 nHeights = property(getNHeights, "I'm the 'nHeights' property.")
947 947 nChannels = property(getNChannels, "I'm the 'nChannel' property.")
948 948 channelIndexList = property(getChannelIndexList, "I'm the 'channelIndexList' property.")
949 949 noise = property(getNoise, "I'm the 'nHeights' property.")
950 950
951 951 ltctime = property(getltctime, "I'm the 'ltctime' property")
952 952 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
953 953
954 954
955 955 class Correlation(JROData):
956 956
957 957 noise = None
958 958
959 959 SNR = None
960 960
961 961 #--------------------------------------------------
962 962
963 963 mode = None
964 964
965 965 split = False
966 966
967 967 data_cf = None
968 968
969 969 lags = None
970 970
971 971 lagRange = None
972 972
973 973 pairsList = None
974 974
975 975 normFactor = None
976 976
977 977 #--------------------------------------------------
978 978
979 979 # calculateVelocity = None
980 980
981 981 nLags = None
982 982
983 983 nPairs = None
984 984
985 985 nAvg = None
986 986
987 987
988 988 def __init__(self):
989 989 '''
990 990 Constructor
991 991 '''
992 992 self.radarControllerHeaderObj = RadarControllerHeader()
993 993
994 994 self.systemHeaderObj = SystemHeader()
995 995
996 996 self.type = "Correlation"
997 997
998 998 self.data = None
999 999
1000 1000 self.dtype = None
1001 1001
1002 1002 self.nProfiles = None
1003 1003
1004 1004 self.heightList = None
1005 1005
1006 1006 self.channelList = None
1007 1007
1008 1008 self.flagNoData = True
1009 1009
1010 1010 self.flagDiscontinuousBlock = False
1011 1011
1012 1012 self.utctime = None
1013 1013
1014 1014 self.timeZone = None
1015 1015
1016 1016 self.dstFlag = None
1017 1017
1018 1018 self.errorCount = None
1019 1019
1020 1020 self.blocksize = None
1021 1021
1022 1022 self.flagDecodeData = False #asumo q la data no esta decodificada
1023 1023
1024 1024 self.flagDeflipData = False #asumo q la data no esta sin flip
1025 1025
1026 1026 self.pairsList = None
1027 1027
1028 1028 self.nPoints = None
1029 1029
1030 1030 def getPairsList(self):
1031 1031
1032 1032 return self.pairsList
1033 1033
1034 1034 def getNoise(self, mode = 2):
1035 1035
1036 1036 indR = numpy.where(self.lagR == 0)[0][0]
1037 1037 indT = numpy.where(self.lagT == 0)[0][0]
1038 1038
1039 1039 jspectra0 = self.data_corr[:,:,indR,:]
1040 1040 jspectra = copy.copy(jspectra0)
1041 1041
1042 1042 num_chan = jspectra.shape[0]
1043 1043 num_hei = jspectra.shape[2]
1044 1044
1045 1045 freq_dc = jspectra.shape[1]/2
1046 1046 ind_vel = numpy.array([-2,-1,1,2]) + freq_dc
1047 1047
1048 1048 if ind_vel[0]<0:
1049 1049 ind_vel[range(0,1)] = ind_vel[range(0,1)] + self.num_prof
1050 1050
1051 1051 if mode == 1:
1052 1052 jspectra[:,freq_dc,:] = (jspectra[:,ind_vel[1],:] + jspectra[:,ind_vel[2],:])/2 #CORRECCION
1053 1053
1054 1054 if mode == 2:
1055 1055
1056 1056 vel = numpy.array([-2,-1,1,2])
1057 1057 xx = numpy.zeros([4,4])
1058 1058
1059 1059 for fil in range(4):
1060 1060 xx[fil,:] = vel[fil]**numpy.asarray(range(4))
1061 1061
1062 1062 xx_inv = numpy.linalg.inv(xx)
1063 1063 xx_aux = xx_inv[0,:]
1064 1064
1065 1065 for ich in range(num_chan):
1066 1066 yy = jspectra[ich,ind_vel,:]
1067 1067 jspectra[ich,freq_dc,:] = numpy.dot(xx_aux,yy)
1068 1068
1069 1069 junkid = jspectra[ich,freq_dc,:]<=0
1070 1070 cjunkid = sum(junkid)
1071 1071
1072 1072 if cjunkid.any():
1073 1073 jspectra[ich,freq_dc,junkid.nonzero()] = (jspectra[ich,ind_vel[1],junkid] + jspectra[ich,ind_vel[2],junkid])/2
1074 1074
1075 1075 noise = jspectra0[:,freq_dc,:] - jspectra[:,freq_dc,:]
1076 1076
1077 1077 return noise
1078 1078
1079 1079 def getTimeInterval(self):
1080 1080
1081 1081 timeInterval = self.ippSeconds * self.nCohInt * self.nProfiles
1082 1082
1083 1083 return timeInterval
1084 1084
1085 1085 def splitFunctions(self):
1086 1086
1087 1087 pairsList = self.pairsList
1088 1088 ccf_pairs = []
1089 1089 acf_pairs = []
1090 1090 ccf_ind = []
1091 1091 acf_ind = []
1092 1092 for l in range(len(pairsList)):
1093 1093 chan0 = pairsList[l][0]
1094 1094 chan1 = pairsList[l][1]
1095 1095
1096 1096 #Obteniendo pares de Autocorrelacion
1097 1097 if chan0 == chan1:
1098 1098 acf_pairs.append(chan0)
1099 1099 acf_ind.append(l)
1100 1100 else:
1101 1101 ccf_pairs.append(pairsList[l])
1102 1102 ccf_ind.append(l)
1103 1103
1104 1104 data_acf = self.data_cf[acf_ind]
1105 1105 data_ccf = self.data_cf[ccf_ind]
1106 1106
1107 1107 return acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf
1108 1108
1109 1109 def getNormFactor(self):
1110 1110 acf_ind, ccf_ind, acf_pairs, ccf_pairs, data_acf, data_ccf = self.splitFunctions()
1111 1111 acf_pairs = numpy.array(acf_pairs)
1112 1112 normFactor = numpy.zeros((self.nPairs,self.nHeights))
1113 1113
1114 1114 for p in range(self.nPairs):
1115 1115 pair = self.pairsList[p]
1116 1116
1117 1117 ch0 = pair[0]
1118 1118 ch1 = pair[1]
1119 1119
1120 1120 ch0_max = numpy.max(data_acf[acf_pairs==ch0,:,:], axis=1)
1121 1121 ch1_max = numpy.max(data_acf[acf_pairs==ch1,:,:], axis=1)
1122 1122 normFactor[p,:] = numpy.sqrt(ch0_max*ch1_max)
1123 1123
1124 1124 return normFactor
1125 1125
1126 1126 timeInterval = property(getTimeInterval, "I'm the 'timeInterval' property")
1127 1127 normFactor = property(getNormFactor, "I'm the 'normFactor property'")
1128 1128
1129 1129 class Parameters(Spectra):
1130 1130
1131 1131 experimentInfo = None #Information about the experiment
1132 1132
1133 1133 #Information from previous data
1134 1134
1135 1135 inputUnit = None #Type of data to be processed
1136 1136
1137 1137 operation = None #Type of operation to parametrize
1138 1138
1139 1139 #normFactor = None #Normalization Factor
1140 1140
1141 1141 groupList = None #List of Pairs, Groups, etc
1142 1142
1143 1143 #Parameters
1144 1144
1145 1145 data_param = None #Parameters obtained
1146 1146
1147 1147 data_pre = None #Data Pre Parametrization
1148 1148
1149 1149 data_SNR = None #Signal to Noise Ratio
1150 1150
1151 1151 # heightRange = None #Heights
1152 1152
1153 1153 abscissaList = None #Abscissa, can be velocities, lags or time
1154 1154
1155 1155 # noise = None #Noise Potency
1156 1156
1157 1157 utctimeInit = None #Initial UTC time
1158 1158
1159 1159 paramInterval = None #Time interval to calculate Parameters in seconds
1160 1160
1161 1161 useLocalTime = True
1162 1162
1163 1163 #Fitting
1164 1164
1165 1165 data_error = None #Error of the estimation
1166 1166
1167 1167 constants = None
1168 1168
1169 1169 library = None
1170 1170
1171 1171 #Output signal
1172 1172
1173 1173 outputInterval = None #Time interval to calculate output signal in seconds
1174 1174
1175 1175 data_output = None #Out signal
1176 1176
1177 1177 nAvg = None
1178 1178
1179 1179 noise_estimation = None
1180 1180
1181 1181
1182 1182 def __init__(self):
1183 1183 '''
1184 1184 Constructor
1185 1185 '''
1186 1186 self.radarControllerHeaderObj = RadarControllerHeader()
1187 1187
1188 1188 self.systemHeaderObj = SystemHeader()
1189 1189
1190 1190 self.type = "Parameters"
1191 1191
1192 1192 def getTimeRange1(self, interval):
1193 1193
1194 1194 datatime = []
1195 1195
1196 1196 if self.useLocalTime:
1197 1197 time1 = self.utctimeInit - self.timeZone*60
1198 1198 else:
1199 1199 time1 = self.utctimeInit
1200 1200
1201 1201 datatime.append(time1)
1202 1202 datatime.append(time1 + interval)
1203 1203 datatime = numpy.array(datatime)
1204 1204
1205 1205 return datatime
1206 1206
1207 1207 def getTimeInterval(self):
1208 1208
1209 1209 if hasattr(self, 'timeInterval1'):
1210 1210 return self.timeInterval1
1211 1211 else:
1212 1212 return self.paramInterval
1213 1213
1214 1214 def getNoise(self):
1215 1215
1216 1216 return self.spc_noise
1217 1217
1218 1218 timeInterval = property(getTimeInterval)
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (1208 lines changed) Show them Hide them
@@ -1,964 +1,782
1 1
2 2 import os
3 import zmq
4 3 import time
5 import numpy
4 import glob
6 5 import datetime
7 import numpy as np
6 from multiprocessing import Process
7
8 import zmq
9 import numpy
8 10 import matplotlib
9 import glob
10 matplotlib.use('TkAgg')
11 11 import matplotlib.pyplot as plt
12 12 from mpl_toolkits.axes_grid1 import make_axes_locatable
13 from matplotlib.ticker import FuncFormatter, LinearLocator
14 from multiprocessing import Process
13 from matplotlib.ticker import FuncFormatter, LinearLocator, MultipleLocator
15 14
16 15 from schainpy.model.proc.jroproc_base import Operation
17
18 plt.ion()
16 from schainpy.utils import log
19 17
20 18 func = lambda x, pos: ('%s') %(datetime.datetime.fromtimestamp(x).strftime('%H:%M'))
21 fromtimestamp = lambda x, mintime : (datetime.datetime.utcfromtimestamp(mintime).replace(hour=(x + 5), minute=0) - d1970).total_seconds()
22 19
20 d1970 = datetime.datetime(1970, 1, 1)
23 21
24 d1970 = datetime.datetime(1970,1,1)
25 22
26 23 class PlotData(Operation, Process):
24 '''
25 Base class for Schain plotting operations
26 '''
27 27
28 28 CODE = 'Figure'
29 29 colormap = 'jro'
30 bgcolor = 'white'
30 31 CONFLATE = False
31 32 __MAXNUMX = 80
32 33 __missing = 1E30
33 34
34 35 def __init__(self, **kwargs):
35 36
36 37 Operation.__init__(self, plot=True, **kwargs)
37 38 Process.__init__(self)
38 39 self.kwargs['code'] = self.CODE
39 40 self.mp = False
40 self.dataOut = None
41 self.isConfig = False
42 self.figure = None
41 self.data = None
42 self.isConfig = False
43 self.figures = []
43 44 self.axes = []
45 self.cb_axes = []
44 46 self.localtime = kwargs.pop('localtime', True)
45 47 self.show = kwargs.get('show', True)
46 48 self.save = kwargs.get('save', False)
47 49 self.colormap = kwargs.get('colormap', self.colormap)
48 50 self.colormap_coh = kwargs.get('colormap_coh', 'jet')
49 51 self.colormap_phase = kwargs.get('colormap_phase', 'RdBu_r')
50 self.showprofile = kwargs.get('showprofile', True)
51 self.title = kwargs.get('wintitle', '')
52 self.colormaps = kwargs.get('colormaps', None)
53 self.bgcolor = kwargs.get('bgcolor', self.bgcolor)
54 self.showprofile = kwargs.get('showprofile', False)
55 self.title = kwargs.get('wintitle', self.CODE.upper())
56 self.cb_label = kwargs.get('cb_label', None)
57 self.cb_labels = kwargs.get('cb_labels', None)
52 58 self.xaxis = kwargs.get('xaxis', 'frequency')
53 59 self.zmin = kwargs.get('zmin', None)
54 60 self.zmax = kwargs.get('zmax', None)
61 self.zlimits = kwargs.get('zlimits', None)
55 62 self.xmin = kwargs.get('xmin', None)
63 if self.xmin is not None:
64 self.xmin += 5
56 65 self.xmax = kwargs.get('xmax', None)
57 66 self.xrange = kwargs.get('xrange', 24)
58 67 self.ymin = kwargs.get('ymin', None)
59 68 self.ymax = kwargs.get('ymax', None)
60 self.__MAXNUMY = kwargs.get('decimation', 5000)
61 self.throttle_value = 5
62 self.times = []
63 #self.interactive = self.kwargs['parent']
69 self.xlabel = kwargs.get('xlabel', None)
70 self.__MAXNUMY = kwargs.get('decimation', 100)
71 self.showSNR = kwargs.get('showSNR', False)
72 self.oneFigure = kwargs.get('oneFigure', True)
73 self.width = kwargs.get('width', None)
74 self.height = kwargs.get('height', None)
75 self.colorbar = kwargs.get('colorbar', True)
76 self.factors = kwargs.get('factors', [1, 1, 1, 1, 1, 1, 1, 1])
77 self.titles = ['' for __ in range(16)]
78
79 def __setup(self):
80 '''
81 Common setup for all figures, here figures and axes are created
82 '''
83
84 self.setup()
85
86 if self.width is None:
87 self.width = 8
64 88
89 self.figures = []
90 self.axes = []
91 self.cb_axes = []
92 self.pf_axes = []
93 self.cmaps = []
94
95 size = '15%' if self.ncols==1 else '30%'
96 pad = '4%' if self.ncols==1 else '8%'
97
98 if self.oneFigure:
99 if self.height is None:
100 self.height = 1.4*self.nrows + 1
101 fig = plt.figure(figsize=(self.width, self.height),
102 edgecolor='k',
103 facecolor='w')
104 self.figures.append(fig)
105 for n in range(self.nplots):
106 ax = fig.add_subplot(self.nrows, self.ncols, n+1)
107 ax.tick_params(labelsize=8)
108 ax.firsttime = True
109 self.axes.append(ax)
110 if self.showprofile:
111 cax = self.__add_axes(ax, size=size, pad=pad)
112 cax.tick_params(labelsize=8)
113 self.pf_axes.append(cax)
114 else:
115 if self.height is None:
116 self.height = 3
117 for n in range(self.nplots):
118 fig = plt.figure(figsize=(self.width, self.height),
119 edgecolor='k',
120 facecolor='w')
121 ax = fig.add_subplot(1, 1, 1)
122 ax.tick_params(labelsize=8)
123 ax.firsttime = True
124 self.figures.append(fig)
125 self.axes.append(ax)
126 if self.showprofile:
127 cax = self.__add_axes(ax, size=size, pad=pad)
128 cax.tick_params(labelsize=8)
129 self.pf_axes.append(cax)
130
131 for n in range(self.nrows):
132 if self.colormaps is not None:
133 cmap = plt.get_cmap(self.colormaps[n])
134 else:
135 cmap = plt.get_cmap(self.colormap)
136 cmap.set_bad(self.bgcolor, 1.)
137 self.cmaps.append(cmap)
138
139 def __add_axes(self, ax, size='30%', pad='8%'):
65 140 '''
66 this new parameter is created to plot data from varius channels at different figures
67 1. crear una lista de figuras donde se puedan plotear las figuras,
68 2. dar las opciones de configuracion a cada figura, estas opciones son iguales para ambas figuras
69 3. probar?
141 Add new axes to the given figure
70 142 '''
71 self.ind_plt_ch = kwargs.get('ind_plt_ch', False)
72 self.figurelist = None
143 divider = make_axes_locatable(ax)
144 nax = divider.new_horizontal(size=size, pad=pad)
145 ax.figure.add_axes(nax)
146 return nax
73 147
74 148
75 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
149 def setup(self):
150 '''
151 This method should be implemented in the child class, the following
152 attributes should be set:
153
154 self.nrows: number of rows
155 self.ncols: number of cols
156 self.nplots: number of plots (channels or pairs)
157 self.ylabel: label for Y axes
158 self.titles: list of axes title
159
160 '''
161 raise(NotImplementedError, 'Implement this method in child class')
76 162
163 def fill_gaps(self, x_buffer, y_buffer, z_buffer):
164 '''
165 Create a masked array for missing data
166 '''
77 167 if x_buffer.shape[0] < 2:
78 168 return x_buffer, y_buffer, z_buffer
79 169
80 170 deltas = x_buffer[1:] - x_buffer[0:-1]
81 x_median = np.median(deltas)
171 x_median = numpy.median(deltas)
82 172
83 index = np.where(deltas > 5*x_median)
173 index = numpy.where(deltas > 5*x_median)
84 174
85 175 if len(index[0]) != 0:
86 176 z_buffer[::, index[0], ::] = self.__missing
87 z_buffer = np.ma.masked_inside(z_buffer,
177 z_buffer = numpy.ma.masked_inside(z_buffer,
88 178 0.99*self.__missing,
89 179 1.01*self.__missing)
90 180
91 181 return x_buffer, y_buffer, z_buffer
92 182
93 183 def decimate(self):
94 184
95 185 # dx = int(len(self.x)/self.__MAXNUMX) + 1
96 186 dy = int(len(self.y)/self.__MAXNUMY) + 1
97 187
98 188 # x = self.x[::dx]
99 189 x = self.x
100 190 y = self.y[::dy]
101 191 z = self.z[::, ::, ::dy]
102
192
103 193 return x, y, z
104 194
105 '''
106 JM:
107 elimana las otras imagenes generadas debido a que lso workers no llegan en orden y le pueden
108 poner otro tiempo a la figura q no necesariamente es el ultimo.
109 Solo se realiza cuando termina la imagen.
110 Problemas:
195 def format(self):
196 '''
197 Set min and max values, labels, ticks and titles
198 '''
111 199
112 File "/home/ci-81/workspace/schainv2.3/schainpy/model/graphics/jroplot_data.py", line 145, in __plot
113 for n, eachfigure in enumerate(self.figurelist):
114 TypeError: 'NoneType' object is not iterable
200 if self.xmin is None:
201 xmin = self.min_time
202 else:
203 if self.xaxis is 'time':
204 dt = datetime.datetime.fromtimestamp(self.min_time)
205 xmin = (datetime.datetime.combine(dt.date(),
206 datetime.time(int(self.xmin), 0, 0))-d1970).total_seconds()
207 else:
208 xmin = self.xmin
115 209
116 '''
117 def deleteanotherfiles(self):
118 figurenames=[]
119 if self.figurelist != None:
120 for n, eachfigure in enumerate(self.figurelist):
121 #add specific name for each channel in channelList
122 ghostfigname = os.path.join(self.save, '{}_{}_{}'.format(self.titles[n].replace(' ',''),self.CODE,
123 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d')))
124 figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n].replace(' ',''),self.CODE,
125 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
126
127 for ghostfigure in glob.glob(ghostfigname+'*'): #ghostfigure will adopt all posible names of figures
128 if ghostfigure != figname:
129 os.remove(ghostfigure)
130 print 'Removing GhostFigures:' , figname
131 else :
132 '''Erasing ghost images for just on******************'''
133 ghostfigname = os.path.join(self.save, '{}_{}'.format(self.CODE,datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d')))
134 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
135 for ghostfigure in glob.glob(ghostfigname+'*'): #ghostfigure will adopt all posible names of figures
136 if ghostfigure != figname:
137 os.remove(ghostfigure)
138 print 'Removing GhostFigures:' , figname
210 if self.xmax is None:
211 xmax = xmin+self.xrange*60*60
212 else:
213 if self.xaxis is 'time':
214 dt = datetime.datetime.fromtimestamp(self.min_time)
215 xmax = (datetime.datetime.combine(dt.date(),
216 datetime.time(int(self.xmax), 0, 0))-d1970).total_seconds()
217 else:
218 xmax = self.xmax
219
220 ymin = self.ymin if self.ymin else numpy.nanmin(self.y)
221 ymax = self.ymax if self.ymax else numpy.nanmax(self.y)
222
223 ystep = 200 if ymax>= 800 else 100 if ymax>=400 else 50 if ymax>=200 else 20
224
225 for n, ax in enumerate(self.axes):
226 if ax.firsttime:
227 ax.set_facecolor(self.bgcolor)
228 ax.yaxis.set_major_locator(MultipleLocator(ystep))
229 if self.xaxis is 'time':
230 ax.xaxis.set_major_formatter(FuncFormatter(func))
231 ax.xaxis.set_major_locator(LinearLocator(9))
232 if self.xlabel is not None:
233 ax.set_xlabel(self.xlabel)
234 ax.set_ylabel(self.ylabel)
235 ax.firsttime = False
236 if self.showprofile:
237 self.pf_axes[n].set_ylim(ymin, ymax)
238 self.pf_axes[n].set_xlim(self.zmin, self.zmax)
239 self.pf_axes[n].set_xlabel('dB')
240 self.pf_axes[n].grid(b=True, axis='x')
241 [tick.set_visible(False) for tick in self.pf_axes[n].get_yticklabels()]
242 if self.colorbar:
243 cb = plt.colorbar(ax.plt, ax=ax, pad=0.02)
244 cb.ax.tick_params(labelsize=8)
245 if self.cb_label:
246 cb.set_label(self.cb_label, size=8)
247 elif self.cb_labels:
248 cb.set_label(self.cb_labels[n], size=8)
249
250 ax.set_title('{} - {} UTC'.format(
251 self.titles[n],
252 datetime.datetime.fromtimestamp(self.max_time).strftime('%H:%M:%S')),
253 size=8)
254 ax.set_xlim(xmin, xmax)
255 ax.set_ylim(ymin, ymax)
256
139 257
140 258 def __plot(self):
141
142 print 'plotting...{}'.format(self.CODE)
143 if self.ind_plt_ch is False : #standard
259 '''
260 '''
261 log.success('Plotting', self.name)
262
263 self.plot()
264 self.format()
265
266 for n, fig in enumerate(self.figures):
267 if self.nrows == 0 or self.nplots == 0:
268 log.warning('No data', self.name)
269 continue
144 270 if self.show:
145 self.figure.show()
146 self.plot()
147 plt.tight_layout()
148 self.figure.canvas.manager.set_window_title('{} {} - {}'.format(self.title, self.CODE.upper(),
149 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
150 else :
151 print 'len(self.figurelist): ',len(self.figurelist)
152 for n, eachfigure in enumerate(self.figurelist):
153 if self.show:
154 eachfigure.show()
155
156 self.plot()
157 eachfigure.tight_layout() # ajuste de cada subplot
158 eachfigure.canvas.manager.set_window_title('{} {} - {}'.format(self.title[n], self.CODE.upper(),
159 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
160
161 # if self.save:
162 # if self.ind_plt_ch is False : #standard
163 # figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
164 # datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
165 # print 'Saving figure: {}'.format(figname)
166 # self.figure.savefig(figname)
167 # else :
168 # for n, eachfigure in enumerate(self.figurelist):
169 # #add specific name for each channel in channelList
170 # figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n],self.CODE,
171 # datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
172 #
173 # print 'Saving figure: {}'.format(figname)
174 # eachfigure.savefig(figname)
175
176 if self.ind_plt_ch is False :
177 self.figure.canvas.draw()
178 else :
179 for eachfigure in self.figurelist:
180 eachfigure.canvas.draw()
181
182 if self.save:
183 if self.ind_plt_ch is False : #standard
184 figname = os.path.join(self.save, '{}_{}.png'.format(self.CODE,
185 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
271 fig.show()
272
273 fig.tight_layout()
274 fig.canvas.manager.set_window_title('{} - {}'.format(self.title,
275 datetime.datetime.fromtimestamp(self.max_time).strftime('%Y/%m/%d')))
276 # fig.canvas.draw()
277
278 if self.save and self.data.ended:
279 channels = range(self.nrows)
280 if self.oneFigure:
281 label = ''
282 else:
283 label = '_{}'.format(channels[n])
284 figname = os.path.join(
285 self.save,
286 '{}{}_{}.png'.format(
287 self.CODE,
288 label,
289 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')
290 )
291 )
186 292 print 'Saving figure: {}'.format(figname)
187 self.figure.savefig(figname)
188 else :
189 for n, eachfigure in enumerate(self.figurelist):
190 #add specific name for each channel in channelList
191 figname = os.path.join(self.save, '{}_{}_{}.png'.format(self.titles[n].replace(' ',''),self.CODE,
192 datetime.datetime.fromtimestamp(self.saveTime).strftime('%y%m%d_%H%M%S')))
193
194 print 'Saving figure: {}'.format(figname)
195 eachfigure.savefig(figname)
196
293 fig.savefig(figname)
197 294
198 295 def plot(self):
199
200 print 'plotting...{}'.format(self.CODE.upper())
201 return
296 '''
297 '''
298 raise(NotImplementedError, 'Implement this method in child class')
202 299
203 300 def run(self):
204 301
205 print '[Starting] {}'.format(self.name)
302 log.success('Starting', self.name)
206 303
207 304 context = zmq.Context()
208 305 receiver = context.socket(zmq.SUB)
209 306 receiver.setsockopt(zmq.SUBSCRIBE, '')
210 307 receiver.setsockopt(zmq.CONFLATE, self.CONFLATE)
211 308
212 309 if 'server' in self.kwargs['parent']:
213 310 receiver.connect('ipc:///tmp/{}.plots'.format(self.kwargs['parent']['server']))
214 311 else:
215 receiver.connect("ipc:///tmp/zmq.plots")
216
217 seconds_passed = 0
312 receiver.connect("ipc:///tmp/zmq.plots")
218 313
219 314 while True:
220 315 try:
221 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)#flags=zmq.NOBLOCK
222 self.started = self.data['STARTED']
223 self.dataOut = self.data['dataOut']
224
225 if (len(self.times) < len(self.data['times']) and not self.started and self.data['ENDED']):
226 continue
227
228 self.times = self.data['times']
229 self.times.sort()
230 self.throttle_value = self.data['throttle']
231 self.min_time = self.times[0]
232 self.max_time = self.times[-1]
316 self.data = receiver.recv_pyobj(flags=zmq.NOBLOCK)
317
318 self.min_time = self.data.times[0]
319 self.max_time = self.data.times[-1]
233 320
234 321 if self.isConfig is False:
235 print 'setting up'
236 self.setup()
322 self.__setup()
237 323 self.isConfig = True
238 self.__plot()
239
240 if self.data['ENDED'] is True:
241 print '********GRAPHIC ENDED********'
242 self.ended = True
243 self.isConfig = False
244 self.__plot()
245 self.deleteanotherfiles() #CLPDG
246 elif seconds_passed >= self.data['throttle']:
247 print 'passed', seconds_passed
248 self.__plot()
249 seconds_passed = 0
324
325 self.__plot()
250 326
251 327 except zmq.Again as e:
252 print 'Waiting for data...'
253 plt.pause(2)
254 seconds_passed += 2
328 log.log('Waiting for data...')
329 if self.data:
330 plt.pause(self.data.throttle)
331 else:
332 time.sleep(2)
255 333
256 334 def close(self):
257 if self.dataOut:
335 if self.data:
258 336 self.__plot()
259 337
260 338
261 339 class PlotSpectraData(PlotData):
340 '''
341 Plot for Spectra data
342 '''
262 343
263 344 CODE = 'spc'
264 colormap = 'jro'
265 CONFLATE = False
345 colormap = 'jro'
266 346
267 347 def setup(self):
268
269 ncolspan = 1
270 colspan = 1
271 self.ncols = int(numpy.sqrt(self.dataOut.nChannels)+0.9)
272 self.nrows = int(self.dataOut.nChannels*1./self.ncols + 0.9)
273 self.width = 3.6*self.ncols
274 self.height = 3.2*self.nrows
275 if self.showprofile:
276 ncolspan = 3
277 colspan = 2
278 self.width += 1.2*self.ncols
348 self.nplots = len(self.data.channels)
349 self.ncols = int(numpy.sqrt(self.nplots)+ 0.9)
350 self.nrows = int((1.0*self.nplots/self.ncols) + 0.9)
351 self.width = 3.4*self.ncols
352 self.height = 3*self.nrows
353 self.cb_label = 'dB'
354 if self.showprofile:
355 self.width += 0.8*self.ncols
279 356
280 357 self.ylabel = 'Range [Km]'
281 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
282
283 if self.figure is None:
284 self.figure = plt.figure(figsize=(self.width, self.height),
285 edgecolor='k',
286 facecolor='w')
287 else:
288 self.figure.clf()
289
290 n = 0
291 for y in range(self.nrows):
292 for x in range(self.ncols):
293 if n >= self.dataOut.nChannels:
294 break
295 ax = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan), 1, colspan)
296 if self.showprofile:
297 ax.ax_profile = plt.subplot2grid((self.nrows, self.ncols*ncolspan), (y, x*ncolspan+colspan), 1, 1)
298
299 ax.firsttime = True
300 self.axes.append(ax)
301 n += 1
302 358
303 359 def plot(self):
304
305 360 if self.xaxis == "frequency":
306 x = self.dataOut.getFreqRange(1)/1000.
307 xlabel = "Frequency (kHz)"
361 x = self.data.xrange[0]
362 self.xlabel = "Frequency (kHz)"
308 363 elif self.xaxis == "time":
309 x = self.dataOut.getAcfRange(1)
310 xlabel = "Time (ms)"
364 x = self.data.xrange[1]
365 self.xlabel = "Time (ms)"
311 366 else:
312 x = self.dataOut.getVelRange(1)
313 xlabel = "Velocity (m/s)"
367 x = self.data.xrange[2]
368 self.xlabel = "Velocity (m/s)"
369
370 if self.CODE == 'spc_mean':
371 x = self.data.xrange[2]
372 self.xlabel = "Velocity (m/s)"
314 373
315 y = self.dataOut.getHeiRange()
316 z = self.data[self.CODE]
374 self.titles = []
317 375
376 y = self.data.heights
377 self.y = y
378 z = self.data['spc']
379
318 380 for n, ax in enumerate(self.axes):
381 noise = self.data['noise'][n][-1]
382 if self.CODE == 'spc_mean':
383 mean = self.data['mean'][n][-1]
319 384 if ax.firsttime:
320 self.xmax = self.xmax if self.xmax else np.nanmax(x)
385 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
321 386 self.xmin = self.xmin if self.xmin else -self.xmax
322 self.ymin = self.ymin if self.ymin else np.nanmin(y)
323 self.ymax = self.ymax if self.ymax else np.nanmax(y)
324 self.zmin = self.zmin if self.zmin else np.nanmin(z)
325 self.zmax = self.zmax if self.zmax else np.nanmax(z)
326 ax.plot = ax.pcolormesh(x, y, z[n].T,
327 vmin=self.zmin,
328 vmax=self.zmax,
329 cmap=plt.get_cmap(self.colormap)
330 )
331 divider = make_axes_locatable(ax)
332 cax = divider.new_horizontal(size='3%', pad=0.05)
333 self.figure.add_axes(cax)
334 plt.colorbar(ax.plot, cax)
335
336 ax.set_xlim(self.xmin, self.xmax)
337 ax.set_ylim(self.ymin, self.ymax)
338
339 ax.set_ylabel(self.ylabel)
340 ax.set_xlabel(xlabel)
341
342 ax.firsttime = False
387 self.zmin = self.zmin if self.zmin else numpy.nanmin(z)
388 self.zmax = self.zmax if self.zmax else numpy.nanmax(z)
389 ax.plt = ax.pcolormesh(x, y, z[n].T,
390 vmin=self.zmin,
391 vmax=self.zmax,
392 cmap=plt.get_cmap(self.colormap)
393 )
343 394
344 395 if self.showprofile:
345 ax.plot_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
346 ax.ax_profile.set_xlim(self.zmin, self.zmax)
347 ax.ax_profile.set_ylim(self.ymin, self.ymax)
348 ax.ax_profile.set_xlabel('dB')
349 ax.ax_profile.grid(b=True, axis='x')
350 ax.plot_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
351 color="k", linestyle="dashed", lw=2)[0]
352 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
396 ax.plt_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], y)[0]
397 ax.plt_noise = self.pf_axes[n].plot(numpy.repeat(noise, len(y)), y,
398 color="k", linestyle="dashed", lw=1)[0]
399 if self.CODE == 'spc_mean':
400 ax.plt_mean = ax.plot(mean, y, color='k')[0]
353 401 else:
354 ax.plot.set_array(z[n].T.ravel())
402 ax.plt.set_array(z[n].T.ravel())
355 403 if self.showprofile:
356 ax.plot_profile.set_data(self.data['rti'][self.max_time][n], y)
357 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
404 ax.plt_profile.set_data(self.data['rti'][n][-1], y)
405 ax.plt_noise.set_data(numpy.repeat(noise, len(y)), y)
406 if self.CODE == 'spc_mean':
407 ax.plt_mean.set_data(mean, y)
358 408
359 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
360 size=8)
409 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
361 410 self.saveTime = self.max_time
362 411
363 412
364 413 class PlotCrossSpectraData(PlotData):
365 414
366 415 CODE = 'cspc'
367 416 zmin_coh = None
368 417 zmax_coh = None
369 418 zmin_phase = None
370 zmax_phase = None
371 CONFLATE = False
419 zmax_phase = None
372 420
373 421 def setup(self):
374 422
375 ncolspan = 1
376 colspan = 1
377 self.ncols = 2
378 self.nrows = self.dataOut.nPairs
379 self.width = 3.6*self.ncols
380 self.height = 3.2*self.nrows
381
423 self.ncols = 4
424 self.nrows = len(self.data.pairs)
425 self.nplots = self.nrows*4
426 self.width = 3.4*self.ncols
427 self.height = 3*self.nrows
382 428 self.ylabel = 'Range [Km]'
383 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
384
385 if self.figure is None:
386 self.figure = plt.figure(figsize=(self.width, self.height),
387 edgecolor='k',
388 facecolor='w')
389 else:
390 self.figure.clf()
391
392 for y in range(self.nrows):
393 for x in range(self.ncols):
394 ax = plt.subplot2grid((self.nrows, self.ncols), (y, x), 1, 1)
395 ax.firsttime = True
396 self.axes.append(ax)
429 self.showprofile = False
397 430
398 431 def plot(self):
399 432
400 433 if self.xaxis == "frequency":
401 x = self.dataOut.getFreqRange(1)/1000.
402 xlabel = "Frequency (kHz)"
434 x = self.data.xrange[0]
435 self.xlabel = "Frequency (kHz)"
403 436 elif self.xaxis == "time":
404 x = self.dataOut.getAcfRange(1)
405 xlabel = "Time (ms)"
437 x = self.data.xrange[1]
438 self.xlabel = "Time (ms)"
406 439 else:
407 x = self.dataOut.getVelRange(1)
408 xlabel = "Velocity (m/s)"
440 x = self.data.xrange[2]
441 self.xlabel = "Velocity (m/s)"
442
443 self.titles = []
409 444
410 y = self.dataOut.getHeiRange()
411 z_coh = self.data['cspc_coh']
412 z_phase = self.data['cspc_phase']
445 y = self.data.heights
446 self.y = y
447 spc = self.data['spc']
448 cspc = self.data['cspc']
413 449
414 450 for n in range(self.nrows):
415 ax = self.axes[2*n]
416 ax1 = self.axes[2*n+1]
451 noise = self.data['noise'][n][-1]
452 pair = self.data.pairs[n]
453 ax = self.axes[4*n]
454 ax3 = self.axes[4*n+3]
417 455 if ax.firsttime:
418 self.xmax = self.xmax if self.xmax else np.nanmax(x)
456 self.xmax = self.xmax if self.xmax else numpy.nanmax(x)
419 457 self.xmin = self.xmin if self.xmin else -self.xmax
420 self.ymin = self.ymin if self.ymin else np.nanmin(y)
421 self.ymax = self.ymax if self.ymax else np.nanmax(y)
422 self.zmin_coh = self.zmin_coh if self.zmin_coh else 0.0
423 self.zmax_coh = self.zmax_coh if self.zmax_coh else 1.0
424 self.zmin_phase = self.zmin_phase if self.zmin_phase else -180
425 self.zmax_phase = self.zmax_phase if self.zmax_phase else 180
426
427 ax.plot = ax.pcolormesh(x, y, z_coh[n].T,
428 vmin=self.zmin_coh,
429 vmax=self.zmax_coh,
430 cmap=plt.get_cmap(self.colormap_coh)
431 )
432 divider = make_axes_locatable(ax)
433 cax = divider.new_horizontal(size='3%', pad=0.05)
434 self.figure.add_axes(cax)
435 plt.colorbar(ax.plot, cax)
436
437 ax.set_xlim(self.xmin, self.xmax)
438 ax.set_ylim(self.ymin, self.ymax)
439
440 ax.set_ylabel(self.ylabel)
441 ax.set_xlabel(xlabel)
442 ax.firsttime = False
443
444 ax1.plot = ax1.pcolormesh(x, y, z_phase[n].T,
445 vmin=self.zmin_phase,
446 vmax=self.zmax_phase,
447 cmap=plt.get_cmap(self.colormap_phase)
448 )
449 divider = make_axes_locatable(ax1)
450 cax = divider.new_horizontal(size='3%', pad=0.05)
451 self.figure.add_axes(cax)
452 plt.colorbar(ax1.plot, cax)
453
454 ax1.set_xlim(self.xmin, self.xmax)
455 ax1.set_ylim(self.ymin, self.ymax)
456
457 ax1.set_ylabel(self.ylabel)
458 ax1.set_xlabel(xlabel)
459 ax1.firsttime = False
458 self.zmin = self.zmin if self.zmin else numpy.nanmin(spc)
459 self.zmax = self.zmax if self.zmax else numpy.nanmax(spc)
460 ax.plt = ax.pcolormesh(x, y, spc[pair[0]].T,
461 vmin=self.zmin,
462 vmax=self.zmax,
463 cmap=plt.get_cmap(self.colormap)
464 )
460 465 else:
461 ax.plot.set_array(z_coh[n].T.ravel())
462 ax1.plot.set_array(z_phase[n].T.ravel())
463
464 ax.set_title('Coherence Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
465 ax1.set_title('Phase Ch{} * Ch{}'.format(self.dataOut.pairsList[n][0], self.dataOut.pairsList[n][1]), size=8)
466 self.saveTime = self.max_time
467
466 ax.plt.set_array(spc[pair[0]].T.ravel())
467 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
468 468
469 class PlotSpectraMeanData(PlotSpectraData):
470
471 CODE = 'spc_mean'
472 colormap = 'jet'
473
474 def plot(self):
475
476 if self.xaxis == "frequency":
477 x = self.dataOut.getFreqRange(1)/1000.
478 xlabel = "Frequency (kHz)"
479 elif self.xaxis == "time":
480 x = self.dataOut.getAcfRange(1)
481 xlabel = "Time (ms)"
482 else:
483 x = self.dataOut.getVelRange(1)
484 xlabel = "Velocity (m/s)"
485
486 y = self.dataOut.getHeiRange()
487 z = self.data['spc']
488 mean = self.data['mean'][self.max_time]
489
490 for n, ax in enumerate(self.axes):
491
492 if ax.firsttime:
493 self.xmax = self.xmax if self.xmax else np.nanmax(x)
494 self.xmin = self.xmin if self.xmin else -self.xmax
495 self.ymin = self.ymin if self.ymin else np.nanmin(y)
496 self.ymax = self.ymax if self.ymax else np.nanmax(y)
497 self.zmin = self.zmin if self.zmin else np.nanmin(z)
498 self.zmax = self.zmax if self.zmax else np.nanmax(z)
499 ax.plt = ax.pcolormesh(x, y, z[n].T,
469 ax = self.axes[4*n+1]
470 if ax.firsttime:
471 ax.plt = ax.pcolormesh(x, y, spc[pair[1]].T,
500 472 vmin=self.zmin,
501 473 vmax=self.zmax,
502 474 cmap=plt.get_cmap(self.colormap)
503 475 )
504 ax.plt_dop = ax.plot(mean[n], y,
505 color='k')[0]
506
507 divider = make_axes_locatable(ax)
508 cax = divider.new_horizontal(size='3%', pad=0.05)
509 self.figure.add_axes(cax)
510 plt.colorbar(ax.plt, cax)
511
512 ax.set_xlim(self.xmin, self.xmax)
513 ax.set_ylim(self.ymin, self.ymax)
514
515 ax.set_ylabel(self.ylabel)
516 ax.set_xlabel(xlabel)
517
518 ax.firsttime = False
519
520 if self.showprofile:
521 ax.plt_profile= ax.ax_profile.plot(self.data['rti'][self.max_time][n], y)[0]
522 ax.ax_profile.set_xlim(self.zmin, self.zmax)
523 ax.ax_profile.set_ylim(self.ymin, self.ymax)
524 ax.ax_profile.set_xlabel('dB')
525 ax.ax_profile.grid(b=True, axis='x')
526 ax.plt_noise = ax.ax_profile.plot(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y,
527 color="k", linestyle="dashed", lw=2)[0]
528 [tick.set_visible(False) for tick in ax.ax_profile.get_yticklabels()]
529 476 else:
530 ax.plt.set_array(z[n].T.ravel())
531 ax.plt_dop.set_data(mean[n], y)
532 if self.showprofile:
533 ax.plt_profile.set_data(self.data['rti'][self.max_time][n], y)
534 ax.plt_noise.set_data(numpy.repeat(self.data['noise'][self.max_time][n], len(y)), y)
477 ax.plt.set_array(spc[pair[1]].T.ravel())
478 self.titles.append('CH {}: {:3.2f}dB'.format(n, noise))
479
480 out = cspc[n]/numpy.sqrt(spc[pair[0]]*spc[pair[1]])
481 coh = numpy.abs(out)
482 phase = numpy.arctan2(out.imag, out.real)*180/numpy.pi
483
484 ax = self.axes[4*n+2]
485 if ax.firsttime:
486 ax.plt = ax.pcolormesh(x, y, coh.T,
487 vmin=0,
488 vmax=1,
489 cmap=plt.get_cmap(self.colormap_coh)
490 )
491 else:
492 ax.plt.set_array(coh.T.ravel())
493 self.titles.append('Coherence Ch{} * Ch{}'.format(pair[0], pair[1]))
535 494
536 ax.set_title('{} - Noise: {:.2f} dB'.format(self.titles[n], self.data['noise'][self.max_time][n]),
537 size=8)
495 ax = self.axes[4*n+3]
496 if ax.firsttime:
497 ax.plt = ax.pcolormesh(x, y, phase.T,
498 vmin=-180,
499 vmax=180,
500 cmap=plt.get_cmap(self.colormap_phase)
501 )
502 else:
503 ax.plt.set_array(phase.T.ravel())
504 self.titles.append('Phase CH{} * CH{}'.format(pair[0], pair[1]))
505
538 506 self.saveTime = self.max_time
539 507
540 508
509 class PlotSpectraMeanData(PlotSpectraData):
510 '''
511 Plot for Spectra and Mean
512 '''
513 CODE = 'spc_mean'
514 colormap = 'jro'
515
516
541 517 class PlotRTIData(PlotData):
518 '''
519 Plot for RTI data
520 '''
542 521
543 522 CODE = 'rti'
544 523 colormap = 'jro'
545 524
546 525 def setup(self):
547 self.ncols = 1
548 self.nrows = self.dataOut.nChannels
549 self.width = 10
550 #TODO : arreglar la altura de la figura, esta hardcodeada.
551 #Se arreglo, testear!
552 if self.ind_plt_ch:
553 self.height = 3.2#*self.nrows if self.nrows<6 else 12
554 else:
555 self.height = 2.2*self.nrows if self.nrows<6 else 12
556
557 '''
558 if self.nrows==1:
559 self.height += 1
560 '''
526 self.xaxis = 'time'
527 self.ncols = 1
528 self.nrows = len(self.data.channels)
529 self.nplots = len(self.data.channels)
561 530 self.ylabel = 'Range [Km]'
562 self.titles = ['Channel {}'.format(x) for x in self.dataOut.channelList]
563
564 '''
565 Logica:
566 1) Si la variable ind_plt_ch es True, va a crear mas de 1 figura
567 2) guardamos "Figures" en una lista y "axes" en otra, quizas se deberia guardar el
568 axis dentro de "Figures" como un diccionario.
569 '''
570 if self.ind_plt_ch is False: #standard mode
571
572 if self.figure is None: #solo para la priemra vez
573 self.figure = plt.figure(figsize=(self.width, self.height),
574 edgecolor='k',
575 facecolor='w')
576 else:
577 self.figure.clf()
578 self.axes = []
579
580
581 for n in range(self.nrows):
582 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
583 #ax = self.figure(n+1)
584 ax.firsttime = True
585 self.axes.append(ax)
586
587 else : #append one figure foreach channel in channelList
588 if self.figurelist == None:
589 self.figurelist = []
590 for n in range(self.nrows):
591 self.figure = plt.figure(figsize=(self.width, self.height),
592 edgecolor='k',
593 facecolor='w')
594 #add always one subplot
595 self.figurelist.append(self.figure)
596
597 else : # cada dia nuevo limpia el axes, pero mantiene el figure
598 for eachfigure in self.figurelist:
599 eachfigure.clf() # eliminaria todas las figuras de la lista?
600 self.axes = []
601
602 for eachfigure in self.figurelist:
603 ax = eachfigure.add_subplot(1,1,1) #solo 1 axis por figura
604 #ax = self.figure(n+1)
605 ax.firsttime = True
606 #Cada figura tiene un distinto puntero
607 self.axes.append(ax)
608 #plt.close(eachfigure)
609
531 self.cb_label = 'dB'
532 self.titles = ['{} Channel {}'.format(self.CODE.upper(), x) for x in range(self.nrows)]
610 533
611 534 def plot(self):
535 self.x = self.data.times
536 self.y = self.data.heights
537 self.z = self.data[self.CODE]
538 self.z = numpy.ma.masked_invalid(self.z)
612 539
613 if self.ind_plt_ch is False: #standard mode
614 self.x = np.array(self.times)
615 self.y = self.dataOut.getHeiRange()
616 self.z = []
617
618 for ch in range(self.nrows):
619 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
620
621 self.z = np.array(self.z)
622 for n, ax in enumerate(self.axes):
623 x, y, z = self.fill_gaps(*self.decimate())
624 if self.xmin is None:
625 xmin = self.min_time
626 else:
627 xmin = fromtimestamp(int(self.xmin), self.min_time)
628 if self.xmax is None:
629 xmax = xmin + self.xrange*60*60
630 else:
631 xmax = xmin + (self.xmax - self.xmin) * 60 * 60
632 self.zmin = self.zmin if self.zmin else np.min(self.z)
633 self.zmax = self.zmax if self.zmax else np.max(self.z)
634 if ax.firsttime:
635 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
636 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
637 plot = ax.pcolormesh(x, y, z[n].T,
638 vmin=self.zmin,
639 vmax=self.zmax,
640 cmap=plt.get_cmap(self.colormap)
641 )
642 divider = make_axes_locatable(ax)
643 cax = divider.new_horizontal(size='2%', pad=0.05)
644 self.figure.add_axes(cax)
645 plt.colorbar(plot, cax)
646 ax.set_ylim(self.ymin, self.ymax)
647 ax.xaxis.set_major_formatter(FuncFormatter(func))
648 ax.xaxis.set_major_locator(LinearLocator(6))
649 ax.set_ylabel(self.ylabel)
650 # if self.xmin is None:
651 # xmin = self.min_time
652 # else:
653 # xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
654 # datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
655
656 ax.set_xlim(xmin, xmax)
657 ax.firsttime = False
658 else:
659 ax.collections.remove(ax.collections[0])
660 ax.set_xlim(xmin, xmax)
661 plot = ax.pcolormesh(x, y, z[n].T,
662 vmin=self.zmin,
663 vmax=self.zmax,
664 cmap=plt.get_cmap(self.colormap)
665 )
666 ax.set_title('{} {}'.format(self.titles[n],
667 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
668 size=8)
669
670 self.saveTime = self.min_time
671 else :
672 self.x = np.array(self.times)
673 self.y = self.dataOut.getHeiRange()
674 self.z = []
675
676 for ch in range(self.nrows):
677 self.z.append([self.data[self.CODE][t][ch] for t in self.times])
678
679 self.z = np.array(self.z)
680 for n, eachfigure in enumerate(self.figurelist): #estaba ax in axes
681
682 x, y, z = self.fill_gaps(*self.decimate())
683 xmin = self.min_time
684 xmax = xmin+self.xrange*60*60
685 self.zmin = self.zmin if self.zmin else np.min(self.z)
686 self.zmax = self.zmax if self.zmax else np.max(self.z)
687 if self.axes[n].firsttime:
688 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
689 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
690 plot = self.axes[n].pcolormesh(x, y, z[n].T,
691 vmin=self.zmin,
692 vmax=self.zmax,
693 cmap=plt.get_cmap(self.colormap)
694 )
695 divider = make_axes_locatable(self.axes[n])
696 cax = divider.new_horizontal(size='2%', pad=0.05)
697 eachfigure.add_axes(cax)
698 #self.figure2.add_axes(cax)
699 plt.colorbar(plot, cax)
700 self.axes[n].set_ylim(self.ymin, self.ymax)
701
702 self.axes[n].xaxis.set_major_formatter(FuncFormatter(func))
703 self.axes[n].xaxis.set_major_locator(LinearLocator(6))
704
705 self.axes[n].set_ylabel(self.ylabel)
706
707 if self.xmin is None:
708 xmin = self.min_time
709 else:
710 xmin = (datetime.datetime.combine(self.dataOut.datatime.date(),
711 datetime.time(self.xmin, 0, 0))-d1970).total_seconds()
712
713 self.axes[n].set_xlim(xmin, xmax)
714 self.axes[n].firsttime = False
715 else:
716 self.axes[n].collections.remove(self.axes[n].collections[0])
717 self.axes[n].set_xlim(xmin, xmax)
718 plot = self.axes[n].pcolormesh(x, y, z[n].T,
719 vmin=self.zmin,
720 vmax=self.zmax,
721 cmap=plt.get_cmap(self.colormap)
722 )
723 self.axes[n].set_title('{} {}'.format(self.titles[n],
724 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
725 size=8)
540 for n, ax in enumerate(self.axes):
541 x, y, z = self.fill_gaps(*self.decimate())
542 self.zmin = self.zmin if self.zmin else numpy.min(self.z)
543 self.zmax = self.zmax if self.zmax else numpy.max(self.z)
544 if ax.firsttime:
545 ax.plt = ax.pcolormesh(x, y, z[n].T,
546 vmin=self.zmin,
547 vmax=self.zmax,
548 cmap=plt.get_cmap(self.colormap)
549 )
550 if self.showprofile:
551 ax.plot_profile= self.pf_axes[n].plot(self.data['rti'][n][-1], self.y)[0]
552 ax.plot_noise = self.pf_axes[n].plot(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y,
553 color="k", linestyle="dashed", lw=1)[0]
554 else:
555 ax.collections.remove(ax.collections[0])
556 ax.plt = ax.pcolormesh(x, y, z[n].T,
557 vmin=self.zmin,
558 vmax=self.zmax,
559 cmap=plt.get_cmap(self.colormap)
560 )
561 if self.showprofile:
562 ax.plot_profile.set_data(self.data['rti'][n][-1], self.y)
563 ax.plot_noise.set_data(numpy.repeat(self.data['noise'][n][-1], len(self.y)), self.y)
726 564
727 self.saveTime = self.min_time
565 self.saveTime = self.min_time
728 566
729 567
730 568 class PlotCOHData(PlotRTIData):
569 '''
570 Plot for Coherence data
571 '''
731 572
732 573 CODE = 'coh'
733 574
734 575 def setup(self):
735
576 self.xaxis = 'time'
736 577 self.ncols = 1
737 self.nrows = self.dataOut.nPairs
738 self.width = 10
739 self.height = 2.2*self.nrows if self.nrows<6 else 12
740 self.ind_plt_ch = False #just for coherence and phase
741 if self.nrows==1:
742 self.height += 1
743 self.ylabel = 'Range [Km]'
744 self.titles = ['{} Ch{} * Ch{}'.format(self.CODE.upper(), x[0], x[1]) for x in self.dataOut.pairsList]
745
746 if self.figure is None:
747 self.figure = plt.figure(figsize=(self.width, self.height),
748 edgecolor='k',
749 facecolor='w')
578 self.nrows = len(self.data.pairs)
579 self.nplots = len(self.data.pairs)
580 self.ylabel = 'Range [Km]'
581 if self.CODE == 'coh':
582 self.cb_label = ''
583 self.titles = ['Coherence Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
750 584 else:
751 self.figure.clf()
752 self.axes = []
585 self.cb_label = 'Degrees'
586 self.titles = ['Phase Map Ch{} * Ch{}'.format(x[0], x[1]) for x in self.data.pairs]
753 587
754 for n in range(self.nrows):
755 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
756 ax.firsttime = True
757 self.axes.append(ax)
588
589 class PlotPHASEData(PlotCOHData):
590 '''
591 Plot for Phase map data
592 '''
593
594 CODE = 'phase'
595 colormap = 'seismic'
758 596
759 597
760 598 class PlotNoiseData(PlotData):
599 '''
600 Plot for noise
601 '''
602
761 603 CODE = 'noise'
762 604
763 605 def setup(self):
764
606 self.xaxis = 'time'
765 607 self.ncols = 1
766 608 self.nrows = 1
767 self.width = 10
768 self.height = 3.2
609 self.nplots = 1
769 610 self.ylabel = 'Intensity [dB]'
770 611 self.titles = ['Noise']
771
772 if self.figure is None:
773 self.figure = plt.figure(figsize=(self.width, self.height),
774 edgecolor='k',
775 facecolor='w')
776 else:
777 self.figure.clf()
778 self.axes = []
779
780 self.ax = self.figure.add_subplot(self.nrows, self.ncols, 1)
781 self.ax.firsttime = True
612 self.colorbar = False
782 613
783 614 def plot(self):
784 615
785 x = self.times
616 x = self.data.times
786 617 xmin = self.min_time
787 618 xmax = xmin+self.xrange*60*60
788 if self.ax.firsttime:
789 for ch in self.dataOut.channelList:
790 y = [self.data[self.CODE][t][ch] for t in self.times]
791 self.ax.plot(x, y, lw=1, label='Ch{}'.format(ch))
792 self.ax.firsttime = False
793 self.ax.xaxis.set_major_formatter(FuncFormatter(func))
794 self.ax.xaxis.set_major_locator(LinearLocator(6))
795 self.ax.set_ylabel(self.ylabel)
619 Y = self.data[self.CODE]
620
621 if self.axes[0].firsttime:
622 for ch in self.data.channels:
623 y = Y[ch]
624 self.axes[0].plot(x, y, lw=1, label='Ch{}'.format(ch))
796 625 plt.legend()
797 626 else:
798 for ch in self.dataOut.channelList:
799 y = [self.data[self.CODE][t][ch] for t in self.times]
800 self.ax.lines[ch].set_data(x, y)
801
802 self.ax.set_xlim(xmin, xmax)
803 self.ax.set_ylim(min(y)-5, max(y)+5)
627 for ch in self.data.channels:
628 y = Y[ch]
629 self.axes[0].lines[ch].set_data(x, y)
630
631 self.ymin = numpy.nanmin(Y) - 5
632 self.ymax = numpy.nanmax(Y) + 5
804 633 self.saveTime = self.min_time
805 634
806 635
807 class PlotWindProfilerData(PlotRTIData):
808
809 CODE = 'wind'
810 colormap = 'seismic'
811
812 def setup(self):
813 self.ncols = 1
814 self.nrows = self.dataOut.data_output.shape[0]
815 self.width = 10
816 self.height = 2.2*self.nrows
817 self.ylabel = 'Height [Km]'
818 self.titles = ['Zonal Wind' ,'Meridional Wind', 'Vertical Wind']
819 self.clabels = ['Velocity (m/s)','Velocity (m/s)','Velocity (cm/s)']
820 self.windFactor = [1, 1, 100]
821
822 if self.figure is None:
823 self.figure = plt.figure(figsize=(self.width, self.height),
824 edgecolor='k',
825 facecolor='w')
826 else:
827 self.figure.clf()
828 self.axes = []
829
830 for n in range(self.nrows):
831 ax = self.figure.add_subplot(self.nrows, self.ncols, n+1)
832 ax.firsttime = True
833 self.axes.append(ax)
834
835 def plot(self):
836
837 self.x = np.array(self.times)
838 self.y = self.dataOut.heightList
839 self.z = []
840
841 for ch in range(self.nrows):
842 self.z.append([self.data['output'][t][ch] for t in self.times])
843
844 self.z = np.array(self.z)
845 self.z = numpy.ma.masked_invalid(self.z)
846
847 cmap=plt.get_cmap(self.colormap)
848 cmap.set_bad('black', 1.)
849
850 for n, ax in enumerate(self.axes):
851 x, y, z = self.fill_gaps(*self.decimate())
852 xmin = self.min_time
853 xmax = xmin+self.xrange*60*60
854 if ax.firsttime:
855 self.ymin = self.ymin if self.ymin else np.nanmin(self.y)
856 self.ymax = self.ymax if self.ymax else np.nanmax(self.y)
857 self.zmax = self.zmax if self.zmax else numpy.nanmax(abs(self.z[:-1, :]))
858 self.zmin = self.zmin if self.zmin else -self.zmax
859
860 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
861 vmin=self.zmin,
862 vmax=self.zmax,
863 cmap=cmap
864 )
865 divider = make_axes_locatable(ax)
866 cax = divider.new_horizontal(size='2%', pad=0.05)
867 self.figure.add_axes(cax)
868 cb = plt.colorbar(plot, cax)
869 cb.set_label(self.clabels[n])
870 ax.set_ylim(self.ymin, self.ymax)
871
872 ax.xaxis.set_major_formatter(FuncFormatter(func))
873 ax.xaxis.set_major_locator(LinearLocator(6))
874
875 ax.set_ylabel(self.ylabel)
876
877 ax.set_xlim(xmin, xmax)
878 ax.firsttime = False
879 else:
880 ax.collections.remove(ax.collections[0])
881 ax.set_xlim(xmin, xmax)
882 plot = ax.pcolormesh(x, y, z[n].T*self.windFactor[n],
883 vmin=self.zmin,
884 vmax=self.zmax,
885 cmap=plt.get_cmap(self.colormap)
886 )
887 ax.set_title('{} {}'.format(self.titles[n],
888 datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')),
889 size=8)
890
891 self.saveTime = self.min_time
892
893
894 636 class PlotSNRData(PlotRTIData):
637 '''
638 Plot for SNR Data
639 '''
640
895 641 CODE = 'snr'
896 642 colormap = 'jet'
897 643
644
898 645 class PlotDOPData(PlotRTIData):
646 '''
647 Plot for DOPPLER Data
648 '''
649
899 650 CODE = 'dop'
900 651 colormap = 'jet'
901 652
902 653
903 class PlotPHASEData(PlotCOHData):
904 CODE = 'phase'
905 colormap = 'seismic'
906
907
908 654 class PlotSkyMapData(PlotData):
655 '''
656 Plot for meteors detection data
657 '''
909 658
910 659 CODE = 'met'
911 660
912 661 def setup(self):
913 662
914 663 self.ncols = 1
915 664 self.nrows = 1
916 665 self.width = 7.2
917 666 self.height = 7.2
918 667
919 668 self.xlabel = 'Zonal Zenith Angle (deg)'
920 669 self.ylabel = 'Meridional Zenith Angle (deg)'
921 670
922 671 if self.figure is None:
923 672 self.figure = plt.figure(figsize=(self.width, self.height),
924 673 edgecolor='k',
925 674 facecolor='w')
926 675 else:
927 676 self.figure.clf()
928 677
929 678 self.ax = plt.subplot2grid((self.nrows, self.ncols), (0, 0), 1, 1, polar=True)
930 679 self.ax.firsttime = True
931 680
932 681
933 682 def plot(self):
934 683
935 arrayParameters = np.concatenate([self.data['param'][t] for t in self.times])
684 arrayParameters = numpy.concatenate([self.data['param'][t] for t in self.data.times])
936 685 error = arrayParameters[:,-1]
937 686 indValid = numpy.where(error == 0)[0]
938 687 finalMeteor = arrayParameters[indValid,:]
939 688 finalAzimuth = finalMeteor[:,3]
940 689 finalZenith = finalMeteor[:,4]
941 690
942 691 x = finalAzimuth*numpy.pi/180
943 692 y = finalZenith
944 693
945 694 if self.ax.firsttime:
946 695 self.ax.plot = self.ax.plot(x, y, 'bo', markersize=5)[0]
947 696 self.ax.set_ylim(0,90)
948 697 self.ax.set_yticks(numpy.arange(0,90,20))
949 698 self.ax.set_xlabel(self.xlabel)
950 699 self.ax.set_ylabel(self.ylabel)
951 700 self.ax.yaxis.labelpad = 40
952 701 self.ax.firsttime = False
953 702 else:
954 703 self.ax.plot.set_data(x, y)
955 704
956 705
957 706 dt1 = datetime.datetime.fromtimestamp(self.min_time).strftime('%y/%m/%d %H:%M:%S')
958 707 dt2 = datetime.datetime.fromtimestamp(self.max_time).strftime('%y/%m/%d %H:%M:%S')
959 708 title = 'Meteor Detection Sky Map\n %s - %s \n Number of events: %5.0f\n' % (dt1,
960 709 dt2,
961 710 len(x))
962 711 self.ax.set_title(title, size=8)
963 712
964 713 self.saveTime = self.max_time
714
715 class PlotParamData(PlotRTIData):
716 '''
717 Plot for data_param object
718 '''
719
720 CODE = 'param'
721 colormap = 'seismic'
722
723 def setup(self):
724 self.xaxis = 'time'
725 self.ncols = 1
726 self.nrows = self.data.shape(self.CODE)[0]
727 self.nplots = self.nrows
728 if self.showSNR:
729 self.nrows += 1
730
731 self.ylabel = 'Height [Km]'
732 self.titles = self.data.parameters \
733 if self.data.parameters else ['Param {}'.format(x) for x in xrange(self.nrows)]
734 if self.showSNR:
735 self.titles.append('SNR')
736
737 def plot(self):
738 self.data.normalize_heights()
739 self.x = self.data.times
740 self.y = self.data.heights
741 if self.showSNR:
742 self.z = numpy.concatenate(
743 (self.data[self.CODE], self.data['snr'])
744 )
745 else:
746 self.z = self.data[self.CODE]
747
748 self.z = numpy.ma.masked_invalid(self.z)
749
750 for n, ax in enumerate(self.axes):
751
752 x, y, z = self.fill_gaps(*self.decimate())
753
754 if ax.firsttime:
755 if self.zlimits is not None:
756 self.zmin, self.zmax = self.zlimits[n]
757 self.zmax = self.zmax if self.zmax is not None else numpy.nanmax(abs(self.z[:-1, :]))
758 self.zmin = self.zmin if self.zmin is not None else -self.zmax
759 ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
760 vmin=self.zmin,
761 vmax=self.zmax,
762 cmap=self.cmaps[n]
763 )
764 else:
765 if self.zlimits is not None:
766 self.zmin, self.zmax = self.zlimits[n]
767 ax.collections.remove(ax.collections[0])
768 ax.plt = ax.pcolormesh(x, y, z[n, :, :].T*self.factors[n],
769 vmin=self.zmin,
770 vmax=self.zmax,
771 cmap=self.cmaps[n]
772 )
773
774 self.saveTime = self.min_time
775
776 class PlotOuputData(PlotParamData):
777 '''
778 Plot data_output object
779 '''
780
781 CODE = 'output'
782 colormap = 'seismic' No newline at end of file
Show file before | Show file after | Hide comments
@@ -1,692 +1,692
1 1 '''
2 2 @author: Daniel Suarez
3 3 '''
4 4
5 5 import os
6 6 import sys
7 7 import glob
8 8 import fnmatch
9 9 import datetime
10 10 import time
11 11 import re
12 12 import h5py
13 13 import numpy
14 14
15 15 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
16 16 from schainpy.model.data.jroamisr import AMISR
17 17
18 18 try:
19 19 from gevent import sleep
20 20 except:
21 21 from time import sleep
22 22
23 23 class RadacHeader():
24 24 def __init__(self, fp):
25 25 header = 'Raw11/Data/RadacHeader'
26 26 self.beamCodeByPulse = fp.get(header+'/BeamCode')
27 27 self.beamCode = fp.get('Raw11/Data/Beamcodes')
28 28 self.code = fp.get(header+'/Code')
29 29 self.frameCount = fp.get(header+'/FrameCount')
30 30 self.modeGroup = fp.get(header+'/ModeGroup')
31 31 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')
32 32 self.pulseCount = fp.get(header+'/PulseCount')
33 33 self.radacTime = fp.get(header+'/RadacTime')
34 34 self.timeCount = fp.get(header+'/TimeCount')
35 35 self.timeStatus = fp.get(header+'/TimeStatus')
36 36
37 37 self.nrecords = self.pulseCount.shape[0] #nblocks
38 38 self.npulses = self.pulseCount.shape[1] #nprofile
39 39 self.nsamples = self.nsamplesPulse[0,0] #ngates
40 40 self.nbeams = self.beamCode.shape[1]
41 41
42 42
43 43 def getIndexRangeToPulse(self, idrecord=0):
44 44 #indexToZero = numpy.where(self.pulseCount.value[idrecord,:]==0)
45 45 #startPulseCountId = indexToZero[0][0]
46 46 #endPulseCountId = startPulseCountId - 1
47 47 #range1 = numpy.arange(startPulseCountId,self.npulses,1)
48 48 #range2 = numpy.arange(0,startPulseCountId,1)
49 49 #return range1, range2
50 50 zero = 0
51 51 npulse = max(self.pulseCount[0,:]+1)-1
52 52 looking_index = numpy.where(self.pulseCount.value[idrecord,:]==npulse)[0]
53 53 getLastIndex = looking_index[-1]
54 54 index_data = numpy.arange(0,getLastIndex+1,1)
55 55 index_buffer = numpy.arange(getLastIndex+1,self.npulses,1)
56 56 return index_data, index_buffer
57 57
58 58 class AMISRReader(ProcessingUnit):
59 59
60 60 path = None
61 61 startDate = None
62 62 endDate = None
63 63 startTime = None
64 64 endTime = None
65 65 walk = None
66 66 isConfig = False
67 67
68 68 def __init__(self):
69 69 self.set = None
70 70 self.subset = None
71 71 self.extension_file = '.h5'
72 72 self.dtc_str = 'dtc'
73 73 self.dtc_id = 0
74 74 self.status = True
75 75 self.isConfig = False
76 76 self.dirnameList = []
77 77 self.filenameList = []
78 78 self.fileIndex = None
79 79 self.flagNoMoreFiles = False
80 80 self.flagIsNewFile = 0
81 81 self.filename = ''
82 82 self.amisrFilePointer = None
83 83 self.radacHeaderObj = None
84 84 self.dataOut = self.__createObjByDefault()
85 85 self.datablock = None
86 86 self.rest_datablock = None
87 87 self.range = None
88 88 self.idrecord_count = 0
89 89 self.profileIndex = 0
90 90 self.index_amisr_sample = None
91 91 self.index_amisr_buffer = None
92 92 self.beamCodeByFrame = None
93 93 self.radacTimeByFrame = None
94 94 #atributos originales tal y como esta en el archivo de datos
95 95 self.beamCodesFromFile = None
96 96 self.radacTimeFromFile = None
97 97 self.rangeFromFile = None
98 98 self.dataByFrame = None
99 99 self.dataset = None
100 100
101 101 self.beamCodeDict = {}
102 102 self.beamRangeDict = {}
103 103
104 104 #experiment cgf file
105 105 self.npulsesint_fromfile = None
106 106 self.recordsperfile_fromfile = None
107 107 self.nbeamcodes_fromfile = None
108 108 self.ngates_fromfile = None
109 109 self.ippSeconds_fromfile = None
110 110 self.frequency_h5file = None
111 111
112 112
113 113 self.__firstFile = True
114 114 self.buffer_radactime = None
115 115
116 116 self.index4_schain_datablock = None
117 117 self.index4_buffer = None
118 118 self.schain_datablock = None
119 119 self.buffer = None
120 120 self.linear_pulseCount = None
121 121 self.npulseByFrame = None
122 122 self.profileIndex_offset = None
123 123 self.timezone = 'ut'
124 124
125 125 self.__waitForNewFile = 20
126 126 self.__filename_online = None
127 127
128 128 def __createObjByDefault(self):
129 129
130 130 dataObj = AMISR()
131 131
132 132 return dataObj
133 133
134 134 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
135 135 self.path = path
136 136 self.startDate = startDate
137 137 self.endDate = endDate
138 138 self.startTime = startTime
139 139 self.endTime = endTime
140 140 self.walk = walk
141 141
142 142 def __checkPath(self):
143 143 if os.path.exists(self.path):
144 144 self.status = 1
145 145 else:
146 146 self.status = 0
147 147 print 'Path:%s does not exists'%self.path
148 148
149 149 return
150 150
151 151 def __selDates(self, amisr_dirname_format):
152 152 try:
153 153 year = int(amisr_dirname_format[0:4])
154 154 month = int(amisr_dirname_format[4:6])
155 155 dom = int(amisr_dirname_format[6:8])
156 156 thisDate = datetime.date(year,month,dom)
157 157
158 158 if (thisDate>=self.startDate and thisDate <= self.endDate):
159 159 return amisr_dirname_format
160 160 except:
161 161 return None
162 162
163 163 def __findDataForDates(self,online=False):
164 164
165 165
166 166
167 167 if not(self.status):
168 168 return None
169 169
170 170 pat = '\d+.\d+'
171 171 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
172 172 dirnameList = filter(lambda x:x!=None,dirnameList)
173 173 dirnameList = [x.string for x in dirnameList]
174 174 if not(online):
175 175 dirnameList = [self.__selDates(x) for x in dirnameList]
176 176 dirnameList = filter(lambda x:x!=None,dirnameList)
177 177 if len(dirnameList)>0:
178 178 self.status = 1
179 179 self.dirnameList = dirnameList
180 180 self.dirnameList.sort()
181 181 else:
182 182 self.status = 0
183 183 return None
184 184
185 185 def __getTimeFromData(self):
186 186 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
187 187 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
188 188
189 189 print 'Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader)
190 190 print '........................................'
191 191 filter_filenameList = []
192 192 self.filenameList.sort()
193 193 for i in range(len(self.filenameList)-1):
194 194 filename = self.filenameList[i]
195 195 fp = h5py.File(filename,'r')
196 196 time_str = fp.get('Time/RadacTimeString')
197 197
198 198 startDateTimeStr_File = time_str[0][0].split('.')[0]
199 199 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
200 200 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
201 201
202 202 endDateTimeStr_File = time_str[-1][-1].split('.')[0]
203 203 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
204 204 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
205 205
206 206 fp.close()
207 207
208 208 if self.timezone == 'lt':
209 209 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
210 210 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
211 211
212 212 if (endDateTime_File>=startDateTime_Reader and endDateTime_File<endDateTime_Reader):
213 213 #self.filenameList.remove(filename)
214 214 filter_filenameList.append(filename)
215 215
216 216 filter_filenameList.sort()
217 217 self.filenameList = filter_filenameList
218 218 return 1
219 219
220 220 def __filterByGlob1(self, dirName):
221 221 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
222 222 filterDict = {}
223 223 filterDict.setdefault(dirName)
224 224 filterDict[dirName] = filter_files
225 225 return filterDict
226 226
227 227 def __getFilenameList(self, fileListInKeys, dirList):
228 228 for value in fileListInKeys:
229 229 dirName = value.keys()[0]
230 230 for file in value[dirName]:
231 231 filename = os.path.join(dirName, file)
232 232 self.filenameList.append(filename)
233 233
234 234
235 235 def __selectDataForTimes(self, online=False):
236 236 #aun no esta implementado el filtro for tiempo
237 237 if not(self.status):
238 238 return None
239 239
240 240 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
241 241
242 242 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
243 243
244 244 self.__getFilenameList(fileListInKeys, dirList)
245 245 if not(online):
246 246 #filtro por tiempo
247 247 if not(self.all):
248 248 self.__getTimeFromData()
249 249
250 250 if len(self.filenameList)>0:
251 251 self.status = 1
252 252 self.filenameList.sort()
253 253 else:
254 254 self.status = 0
255 255 return None
256 256
257 257 else:
258 258 #get the last file - 1
259 259 self.filenameList = [self.filenameList[-2]]
260 260
261 261 new_dirnameList = []
262 262 for dirname in self.dirnameList:
263 263 junk = numpy.array([dirname in x for x in self.filenameList])
264 264 junk_sum = junk.sum()
265 265 if junk_sum > 0:
266 266 new_dirnameList.append(dirname)
267 267 self.dirnameList = new_dirnameList
268 268 return 1
269 269
270 def __searchFilesOnline(self,
270 def searchFilesOnLine(self,
271 271 path,
272 272 walk=True):
273 273
274 274 startDate = datetime.datetime.utcnow().date()
275 275 endDate = datetime.datetime.utcnow().date()
276 276
277 277 self.__setParameters(path=path, startDate=startDate, endDate=endDate, walk=walk)
278 278
279 279 self.__checkPath()
280 280
281 281 self.__findDataForDates(online=True)
282 282
283 283 self.dirnameList = [self.dirnameList[-1]]
284 284
285 285 self.__selectDataForTimes(online=True)
286 286
287 287 return
288 288
289 289
290 def __searchFilesOffline(self,
290 def searchFilesOffLine(self,
291 291 path,
292 292 startDate,
293 293 endDate,
294 294 startTime=datetime.time(0,0,0),
295 295 endTime=datetime.time(23,59,59),
296 296 walk=True):
297 297
298 298 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
299 299
300 300 self.__checkPath()
301 301
302 302 self.__findDataForDates()
303 303
304 304 self.__selectDataForTimes()
305 305
306 306 for i in range(len(self.filenameList)):
307 307 print "%s" %(self.filenameList[i])
308 308
309 309 return
310 310
311 311 def __setNextFileOffline(self):
312 312 idFile = self.fileIndex
313 313
314 314 while (True):
315 315 idFile += 1
316 316 if not(idFile < len(self.filenameList)):
317 317 self.flagNoMoreFiles = 1
318 318 print "No more Files"
319 319 return 0
320 320
321 321 filename = self.filenameList[idFile]
322 322
323 323 amisrFilePointer = h5py.File(filename,'r')
324 324
325 325 break
326 326
327 327 self.flagIsNewFile = 1
328 328 self.fileIndex = idFile
329 329 self.filename = filename
330 330
331 331 self.amisrFilePointer = amisrFilePointer
332 332
333 333 print "Setting the file: %s"%self.filename
334 334
335 335 return 1
336 336
337 337
338 338 def __setNextFileOnline(self):
339 339 filename = self.filenameList[0]
340 340 if self.__filename_online != None:
341 341 self.__selectDataForTimes(online=True)
342 342 filename = self.filenameList[0]
343 343 while self.__filename_online == filename:
344 344 print 'waiting %d seconds to get a new file...'%(self.__waitForNewFile)
345 345 sleep(self.__waitForNewFile)
346 346 self.__selectDataForTimes(online=True)
347 347 filename = self.filenameList[0]
348 348
349 349 self.__filename_online = filename
350 350
351 351 self.amisrFilePointer = h5py.File(filename,'r')
352 352 self.flagIsNewFile = 1
353 353 self.filename = filename
354 354 print "Setting the file: %s"%self.filename
355 355 return 1
356 356
357 357
358 358 def __readHeader(self):
359 359 self.radacHeaderObj = RadacHeader(self.amisrFilePointer)
360 360
361 361 #update values from experiment cfg file
362 362 if self.radacHeaderObj.nrecords == self.recordsperfile_fromfile:
363 363 self.radacHeaderObj.nrecords = self.recordsperfile_fromfile
364 364 self.radacHeaderObj.nbeams = self.nbeamcodes_fromfile
365 365 self.radacHeaderObj.npulses = self.npulsesint_fromfile
366 366 self.radacHeaderObj.nsamples = self.ngates_fromfile
367 367
368 368 #looking index list for data
369 369 start_index = self.radacHeaderObj.pulseCount[0,:][0]
370 370 end_index = self.radacHeaderObj.npulses
371 371 range4data = range(start_index, end_index)
372 372 self.index4_schain_datablock = numpy.array(range4data)
373 373
374 374 buffer_start_index = 0
375 375 buffer_end_index = self.radacHeaderObj.pulseCount[0,:][0]
376 376 range4buffer = range(buffer_start_index, buffer_end_index)
377 377 self.index4_buffer = numpy.array(range4buffer)
378 378
379 379 self.linear_pulseCount = numpy.array(range4data + range4buffer)
380 380 self.npulseByFrame = max(self.radacHeaderObj.pulseCount[0,:]+1)
381 381
382 382 #get tuning frequency
383 383 frequency_h5file_dataset = self.amisrFilePointer.get('Rx'+'/TuningFrequency')
384 384 self.frequency_h5file = frequency_h5file_dataset[0,0]
385 385
386 386 self.flagIsNewFile = 1
387 387
388 388 def __getBeamCode(self):
389 389 self.beamCodeDict = {}
390 390 self.beamRangeDict = {}
391 391
392 392 beamCodeMap = self.amisrFilePointer.get('Setup/BeamcodeMap')
393 393
394 394 for i in range(len(self.radacHeaderObj.beamCode[0,:])):
395 395 self.beamCodeDict.setdefault(i)
396 396 self.beamRangeDict.setdefault(i)
397 397 beamcodeValue = self.radacHeaderObj.beamCode[0,i]
398 398 beamcodeIndex = numpy.where(beamCodeMap[:,0] == beamcodeValue)[0][0]
399 399 x = beamCodeMap[beamcodeIndex][1]
400 400 y = beamCodeMap[beamcodeIndex][2]
401 401 z = beamCodeMap[beamcodeIndex][3]
402 402 self.beamCodeDict[i] = [beamcodeValue, x, y, z]
403 403
404 404 just4record0 = self.radacHeaderObj.beamCodeByPulse[0,:]
405 405
406 406 for i in range(len(self.beamCodeDict.values())):
407 407 xx = numpy.where(just4record0==self.beamCodeDict.values()[i][0])
408 408 indexPulseByBeam = self.linear_pulseCount[xx[0]]
409 409 self.beamRangeDict[i] = indexPulseByBeam
410 410
411 411 def __getExpParameters(self):
412 412 if not(self.status):
413 413 return None
414 414
415 415 experimentCfgPath = os.path.join(self.path, self.dirnameList[0], 'Setup')
416 416
417 417 expFinder = glob.glob1(experimentCfgPath,'*.exp')
418 418 if len(expFinder)== 0:
419 419 self.status = 0
420 420 return None
421 421
422 422 experimentFilename = os.path.join(experimentCfgPath,expFinder[0])
423 423
424 424 f = open(experimentFilename)
425 425 lines = f.readlines()
426 426 f.close()
427 427
428 428 parmsList = ['npulsesint*','recordsperfile*','nbeamcodes*','ngates*']
429 429 filterList = [fnmatch.filter(lines, x) for x in parmsList]
430 430
431 431
432 432 values = [re.sub(r'\D',"",x[0]) for x in filterList]
433 433
434 434 self.npulsesint_fromfile = int(values[0])
435 435 self.recordsperfile_fromfile = int(values[1])
436 436 self.nbeamcodes_fromfile = int(values[2])
437 437 self.ngates_fromfile = int(values[3])
438 438
439 439 tufileFinder = fnmatch.filter(lines, 'tufile=*')
440 440 tufile = tufileFinder[0].split('=')[1].split('\n')[0]
441 441 tufile = tufile.split('\r')[0]
442 442 tufilename = os.path.join(experimentCfgPath,tufile)
443 443
444 444 f = open(tufilename)
445 445 lines = f.readlines()
446 446 f.close()
447 447 self.ippSeconds_fromfile = float(lines[1].split()[2])/1E6
448 448
449 449
450 450 self.status = 1
451 451
452 452 def __setIdsAndArrays(self):
453 453 self.dataByFrame = self.__setDataByFrame()
454 454 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[0, :]
455 455 self.readRanges()
456 456 self.index_amisr_sample, self.index_amisr_buffer = self.radacHeaderObj.getIndexRangeToPulse(0)
457 457 self.radacTimeByFrame = numpy.zeros(self.radacHeaderObj.npulses)
458 458 if len(self.index_amisr_buffer) > 0:
459 459 self.buffer_radactime = numpy.zeros_like(self.radacTimeByFrame)
460 460
461 461
462 462 def __setNextFile(self,online=False):
463 463
464 464 if not(online):
465 465 newFile = self.__setNextFileOffline()
466 466 else:
467 467 newFile = self.__setNextFileOnline()
468 468
469 469 if not(newFile):
470 470 return 0
471 471
472 472 self.__readHeader()
473 473
474 474 if self.__firstFile:
475 475 self.__setIdsAndArrays()
476 476 self.__firstFile = False
477 477
478 478 self.__getBeamCode()
479 479 self.readDataBlock()
480 480
481 481
482 482 def setup(self,path=None,
483 483 startDate=None,
484 484 endDate=None,
485 485 startTime=datetime.time(0,0,0),
486 486 endTime=datetime.time(23,59,59),
487 487 walk=True,
488 488 timezone='ut',
489 489 all=0,
490 490 online=False):
491 491
492 492 self.timezone = timezone
493 493 self.all = all
494 494 self.online = online
495 495 if not(online):
496 496 #Busqueda de archivos offline
497 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 498 else:
499 self.__searchFilesOnline(path, walk)
499 self.searchFilesOnLine(path, walk)
500 500
501 501 if not(self.filenameList):
502 502 print "There is no files into the folder: %s"%(path)
503 503
504 504 sys.exit(-1)
505 505
506 506 self.__getExpParameters()
507 507
508 508 self.fileIndex = -1
509 509
510 510 self.__setNextFile(online)
511 511
512 512 # first_beamcode = self.radacHeaderObj.beamCodeByPulse[0,0]
513 513 # index = numpy.where(self.radacHeaderObj.beamCodeByPulse[0,:]!=first_beamcode)[0][0]
514 514 self.profileIndex_offset = self.radacHeaderObj.pulseCount[0,:][0]
515 515 self.profileIndex = self.profileIndex_offset
516 516
517 517 def readRanges(self):
518 518 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Range')
519 519
520 520 self.rangeFromFile = numpy.reshape(dataset.value,(-1))
521 521 return self.rangeFromFile
522 522
523 523
524 524 def readRadacTime(self,idrecord, range1, range2):
525 525 self.radacTimeFromFile = self.radacHeaderObj.radacTime.value
526 526
527 527 radacTimeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
528 528 #radacTimeByFrame = dataset[idrecord - 1,range1]
529 529 #radacTimeByFrame = dataset[idrecord,range2]
530 530
531 531 return radacTimeByFrame
532 532
533 533 def readBeamCode(self, idrecord, range1, range2):
534 534 dataset = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode')
535 535 beamcodeByFrame = numpy.zeros((self.radacHeaderObj.npulses))
536 536 self.beamCodesFromFile = dataset.value
537 537
538 538 #beamcodeByFrame[range1] = dataset[idrecord - 1, range1]
539 539 #beamcodeByFrame[range2] = dataset[idrecord, range2]
540 540 beamcodeByFrame[range1] = dataset[idrecord, range1]
541 541 beamcodeByFrame[range2] = dataset[idrecord, range2]
542 542
543 543 return beamcodeByFrame
544 544
545 545
546 546 def __setDataByFrame(self):
547 547 ndata = 2 # porque es complejo
548 548 dataByFrame = numpy.zeros((self.radacHeaderObj.npulses, self.radacHeaderObj.nsamples, ndata))
549 549 return dataByFrame
550 550
551 551 def __readDataSet(self):
552 552 dataset = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
553 553 return dataset
554 554
555 555 def __setDataBlock(self,):
556 556 real = self.dataByFrame[:,:,0] #asumo que 0 es real
557 557 imag = self.dataByFrame[:,:,1] #asumo que 1 es imaginario
558 558 datablock = real + imag*1j #armo el complejo
559 559 return datablock
560 560
561 561 def readSamples_version1(self,idrecord):
562 562 #estas tres primeras lineas solo se deben ejecutar una vez
563 563 if self.flagIsNewFile:
564 564 #reading dataset
565 565 self.dataset = self.__readDataSet()
566 566 self.flagIsNewFile = 0
567 567
568 568 if idrecord == 0:
569 569 self.dataByFrame[self.index4_schain_datablock, : ,:] = self.dataset[0, self.index_amisr_sample,:,:]
570 570 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[0, self.index_amisr_sample]
571 571 datablock = self.__setDataBlock()
572 572 if len(self.index_amisr_buffer) > 0:
573 573 self.buffer = self.dataset[0, self.index_amisr_buffer,:,:]
574 574 self.buffer_radactime = self.radacHeaderObj.radacTime[0, self.index_amisr_buffer]
575 575
576 576 return datablock
577 577 if len(self.index_amisr_buffer) > 0:
578 578 self.dataByFrame[self.index4_buffer,:,:] = self.buffer.copy()
579 579 self.radacTimeByFrame[self.index4_buffer] = self.buffer_radactime.copy()
580 580 self.dataByFrame[self.index4_schain_datablock,:,:] = self.dataset[idrecord, self.index_amisr_sample,:,:]
581 581 self.radacTimeByFrame[self.index4_schain_datablock] = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_sample]
582 582 datablock = self.__setDataBlock()
583 583 if len(self.index_amisr_buffer) > 0:
584 584 self.buffer = self.dataset[idrecord, self.index_amisr_buffer, :, :]
585 585 self.buffer_radactime = self.radacHeaderObj.radacTime[idrecord, self.index_amisr_buffer]
586 586
587 587 return datablock
588 588
589 589
590 590 def readSamples(self,idrecord):
591 591 if self.flagIsNewFile:
592 592 self.dataByFrame = self.__setDataByFrame()
593 593 self.beamCodeByFrame = self.amisrFilePointer.get('Raw11/Data/RadacHeader/BeamCode').value[idrecord, :]
594 594
595 595 #reading ranges
596 596 self.readRanges()
597 597 #reading dataset
598 598 self.dataset = self.__readDataSet()
599 599
600 600 self.flagIsNewFile = 0
601 601 self.radacTimeByFrame = self.radacHeaderObj.radacTime.value[idrecord, :]
602 602 self.dataByFrame = self.dataset[idrecord, :, :, :]
603 603 datablock = self.__setDataBlock()
604 604 return datablock
605 605
606 606
607 607 def readDataBlock(self):
608 608
609 609 self.datablock = self.readSamples_version1(self.idrecord_count)
610 610 #self.datablock = self.readSamples(self.idrecord_count)
611 611 #print 'record:', self.idrecord_count
612 612
613 613 self.idrecord_count += 1
614 614 self.profileIndex = 0
615 615
616 616 if self.idrecord_count >= self.radacHeaderObj.nrecords:
617 617 self.idrecord_count = 0
618 618 self.flagIsNewFile = 1
619 619
620 620 def readNextBlock(self):
621 621
622 622 self.readDataBlock()
623 623
624 624 if self.flagIsNewFile:
625 625 self.__setNextFile(self.online)
626 626 pass
627 627
628 628 def __hasNotDataInBuffer(self):
629 629 #self.radacHeaderObj.npulses debe ser otra variable para considerar el numero de pulsos a tomar en el primer y ultimo record
630 630 if self.profileIndex >= self.radacHeaderObj.npulses:
631 631 return 1
632 632 return 0
633 633
634 634 def printUTC(self):
635 635 print self.dataOut.utctime
636 636 print ''
637 637
638 638 def setObjProperties(self):
639 639
640 640 self.dataOut.heightList = self.rangeFromFile/1000.0 #km
641 641 self.dataOut.nProfiles = self.radacHeaderObj.npulses
642 642 self.dataOut.nRecords = self.radacHeaderObj.nrecords
643 643 self.dataOut.nBeams = self.radacHeaderObj.nbeams
644 644 self.dataOut.ippSeconds = self.ippSeconds_fromfile
645 645 # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
646 646 self.dataOut.frequency = self.frequency_h5file
647 647 self.dataOut.npulseByFrame = self.npulseByFrame
648 648 self.dataOut.nBaud = None
649 649 self.dataOut.nCode = None
650 650 self.dataOut.code = None
651 651
652 652 self.dataOut.beamCodeDict = self.beamCodeDict
653 653 self.dataOut.beamRangeDict = self.beamRangeDict
654 654
655 655 if self.timezone == 'lt':
656 656 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
657 657 else:
658 658 self.dataOut.timeZone = 0 #by default time is UTC
659 659
660 660 def getData(self):
661 661
662 662 if self.flagNoMoreFiles:
663 663 self.dataOut.flagNoData = True
664 664 print 'Process finished'
665 665 return 0
666 666
667 667 if self.__hasNotDataInBuffer():
668 668 self.readNextBlock()
669 669
670 670
671 671 if self.datablock is None: # setear esta condicion cuando no hayan datos por leers
672 672 self.dataOut.flagNoData = True
673 673 return 0
674 674
675 675 self.dataOut.data = numpy.reshape(self.datablock[self.profileIndex,:],(1,-1))
676 676
677 677 self.dataOut.utctime = self.radacTimeByFrame[self.profileIndex]
678 678 self.dataOut.profileIndex = self.profileIndex
679 679 self.dataOut.flagNoData = False
680 680
681 681 self.profileIndex += 1
682 682
683 683 return self.dataOut.data
684 684
685 685
686 686 def run(self, **kwargs):
687 687 if not(self.isConfig):
688 688 self.setup(**kwargs)
689 689 self.setObjProperties()
690 690 self.isConfig = True
691 691
692 692 self.getData()
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (503 lines changed) Show them Hide them
@@ -1,1854 +1,1795
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 14 import traceback
15 15 import zmq
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 24
25 25 LOCALTIME = True
26 26
27 27 def isNumber(cad):
28 28 """
29 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30 30
31 31 Excepciones:
32 32 Si un determinado string no puede ser convertido a numero
33 33 Input:
34 34 str, string al cual se le analiza para determinar si convertible a un numero o no
35 35
36 36 Return:
37 37 True : si el string es uno numerico
38 38 False : no es un string numerico
39 39 """
40 40 try:
41 41 float( cad )
42 42 return True
43 43 except:
44 44 return False
45 45
46 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 47 """
48 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49 49
50 50 Inputs:
51 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52 52
53 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 56 segundos contados desde 01/01/1970.
57 57
58 58 Return:
59 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 60 fecha especificado, de lo contrario retorna False.
61 61
62 62 Excepciones:
63 63 Si el archivo no existe o no puede ser abierto
64 64 Si la cabecera no puede ser leida.
65 65
66 66 """
67 67 basicHeaderObj = BasicHeader(LOCALTIME)
68 68
69 69 try:
70 70 fp = open(filename,'rb')
71 71 except IOError:
72 72 print "The file %s can't be opened" %(filename)
73 73 return 0
74 74
75 75 sts = basicHeaderObj.read(fp)
76 76 fp.close()
77 77
78 78 if not(sts):
79 79 print "Skipping the file %s because it has not a valid header" %(filename)
80 80 return 0
81 81
82 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 83 return 0
84 84
85 85 return 1
86 86
87 87 def isTimeInRange(thisTime, startTime, endTime):
88 88
89 89 if endTime >= startTime:
90 90 if (thisTime < startTime) or (thisTime > endTime):
91 91 return 0
92 92
93 93 return 1
94 94 else:
95 95 if (thisTime < startTime) and (thisTime > endTime):
96 96 return 0
97 97
98 98 return 1
99 99
100 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 101 """
102 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103 103
104 104 Inputs:
105 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106 106
107 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108 108
109 109 endDate : fecha final del rango seleccionado en formato datetime.date
110 110
111 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112 112
113 113 endTime : tiempo final del rango seleccionado en formato datetime.time
114 114
115 115 Return:
116 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 117 fecha especificado, de lo contrario retorna False.
118 118
119 119 Excepciones:
120 120 Si el archivo no existe o no puede ser abierto
121 121 Si la cabecera no puede ser leida.
122 122
123 123 """
124 124
125 125
126 126 try:
127 127 fp = open(filename,'rb')
128 128 except IOError:
129 129 print "The file %s can't be opened" %(filename)
130 130 return None
131 131
132 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 133 systemHeaderObj = SystemHeader()
134 134 radarControllerHeaderObj = RadarControllerHeader()
135 135 processingHeaderObj = ProcessingHeader()
136 136
137 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 138
139 139 sts = firstBasicHeaderObj.read(fp)
140 140
141 141 if not(sts):
142 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 143 return None
144 144
145 145 if not systemHeaderObj.read(fp):
146 146 return None
147 147
148 148 if not radarControllerHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not processingHeaderObj.read(fp):
152 152 return None
153 153
154 154 filesize = os.path.getsize(filename)
155 155
156 156 offset = processingHeaderObj.blockSize + 24 #header size
157 157
158 158 if filesize <= offset:
159 159 print "[Reading] %s: This file has not enough data" %filename
160 160 return None
161 161
162 162 fp.seek(-offset, 2)
163 163
164 164 sts = lastBasicHeaderObj.read(fp)
165 165
166 166 fp.close()
167 167
168 168 thisDatetime = lastBasicHeaderObj.datatime
169 169 thisTime_last_block = thisDatetime.time()
170 170
171 171 thisDatetime = firstBasicHeaderObj.datatime
172 172 thisDate = thisDatetime.date()
173 173 thisTime_first_block = thisDatetime.time()
174 174
175 175 #General case
176 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 177 #-----------o----------------------------o-----------
178 178 # startTime endTime
179 179
180 180 if endTime >= startTime:
181 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 182 return None
183 183
184 184 return thisDatetime
185 185
186 186 #If endTime < startTime then endTime belongs to the next day
187 187
188 188
189 189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 190 #-----------o----------------------------o-----------
191 191 # endTime startTime
192 192
193 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 194 return None
195 195
196 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 197 return None
198 198
199 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 200 return None
201 201
202 202 return thisDatetime
203 203
204 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 205 """
206 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 207
208 208 Inputs:
209 209 folder : nombre completo del directorio.
210 210 Su formato deberia ser "/path_root/?YYYYDDD"
211 211
212 212 siendo:
213 213 YYYY : Anio (ejemplo 2015)
214 214 DDD : Dia del anio (ejemplo 305)
215 215
216 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 217
218 218 endDate : fecha final del rango seleccionado en formato datetime.date
219 219
220 220 Return:
221 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 222 fecha especificado, de lo contrario retorna False.
223 223 Excepciones:
224 224 Si el directorio no tiene el formato adecuado
225 225 """
226 226
227 227 basename = os.path.basename(folder)
228 228
229 229 if not isRadarFolder(basename):
230 230 print "The folder %s has not the rigth format" %folder
231 231 return 0
232 232
233 233 if startDate and endDate:
234 234 thisDate = getDateFromRadarFolder(basename)
235 235
236 236 if thisDate < startDate:
237 237 return 0
238 238
239 239 if thisDate > endDate:
240 240 return 0
241 241
242 242 return 1
243 243
244 244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 245 """
246 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247 247
248 248 Inputs:
249 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250 250
251 251 Su formato deberia ser "?YYYYDDDsss"
252 252
253 253 siendo:
254 254 YYYY : Anio (ejemplo 2015)
255 255 DDD : Dia del anio (ejemplo 305)
256 256 sss : set
257 257
258 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259 259
260 260 endDate : fecha final del rango seleccionado en formato datetime.date
261 261
262 262 Return:
263 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 264 fecha especificado, de lo contrario retorna False.
265 265 Excepciones:
266 266 Si el archivo no tiene el formato adecuado
267 267 """
268 268
269 269 basename = os.path.basename(filename)
270 270
271 271 if not isRadarFile(basename):
272 272 print "The filename %s has not the rigth format" %filename
273 273 return 0
274 274
275 275 if startDate and endDate:
276 276 thisDate = getDateFromRadarFile(basename)
277 277
278 278 if thisDate < startDate:
279 279 return 0
280 280
281 281 if thisDate > endDate:
282 282 return 0
283 283
284 284 return 1
285 285
286 286 def getFileFromSet(path, ext, set):
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294 try:
295 295 year = int(thisFile[1:5])
296 296 doy = int(thisFile[5:8])
297 297 except:
298 298 continue
299 299
300 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 301 continue
302 302
303 303 validFilelist.append(thisFile)
304 304
305 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306 306
307 307 if len(myfile)!= 0:
308 308 return myfile[0]
309 309 else:
310 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 311 print 'the filename %s does not exist'%filename
312 312 print '...going to the last file: '
313 313
314 314 if validFilelist:
315 315 validFilelist = sorted( validFilelist, key=str.lower )
316 316 return validFilelist[-1]
317 317
318 318 return None
319 319
320 320 def getlastFileFromPath(path, ext):
321 321 """
322 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324 324
325 325 Input:
326 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 327 ext : extension de los files contenidos en una carpeta
328 328
329 329 Return:
330 330 El ultimo file de una determinada carpeta, no se considera el path.
331 331 """
332 332 validFilelist = []
333 333 fileList = os.listdir(path)
334 334
335 335 # 0 1234 567 89A BCDE
336 336 # H YYYY DDD SSS .ext
337 337
338 338 for thisFile in fileList:
339 339
340 340 year = thisFile[1:5]
341 341 if not isNumber(year):
342 342 continue
343 343
344 344 doy = thisFile[5:8]
345 345 if not isNumber(doy):
346 346 continue
347 347
348 348 year = int(year)
349 349 doy = int(doy)
350 350
351 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 352 continue
353 353
354 354 validFilelist.append(thisFile)
355 355
356 356 if validFilelist:
357 357 validFilelist = sorted( validFilelist, key=str.lower )
358 358 return validFilelist[-1]
359 359
360 360 return None
361 361
362 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 363 """
364 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 366 el path exacto de un determinado file.
367 367
368 368 Example :
369 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370 370
371 371 Entonces la funcion prueba con las siguientes combinaciones
372 372 .../.../y2009307367.ext
373 373 .../.../Y2009307367.ext
374 374 .../.../x2009307/y2009307367.ext
375 375 .../.../x2009307/Y2009307367.ext
376 376 .../.../X2009307/y2009307367.ext
377 377 .../.../X2009307/Y2009307367.ext
378 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379 379
380 380 Return:
381 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 383 para el filename
384 384 """
385 385 fullfilename = None
386 386 find_flag = False
387 387 filename = None
388 388
389 389 prefixDirList = [None,'d','D']
390 390 if ext.lower() == ".r": #voltage
391 391 prefixFileList = ['d','D']
392 392 elif ext.lower() == ".pdata": #spectra
393 393 prefixFileList = ['p','P']
394 394 else:
395 395 return None, filename
396 396
397 397 #barrido por las combinaciones posibles
398 398 for prefixDir in prefixDirList:
399 399 thispath = path
400 400 if prefixDir != None:
401 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 402 if foldercounter == 0:
403 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 404 else:
405 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409 409
410 410 if os.path.exists( fullfilename ): #verifico que exista
411 411 find_flag = True
412 412 break
413 413 if find_flag:
414 414 break
415 415
416 416 if not(find_flag):
417 417 return None, filename
418 418
419 419 return fullfilename, filename
420 420
421 421 def isRadarFolder(folder):
422 422 try:
423 423 year = int(folder[1:5])
424 424 doy = int(folder[5:8])
425 425 except:
426 426 return 0
427 427
428 428 return 1
429 429
430 430 def isRadarFile(file):
431 431 try:
432 432 year = int(file[1:5])
433 433 doy = int(file[5:8])
434 434 set = int(file[8:11])
435 435 except:
436 436 return 0
437 437
438 438 return 1
439 439
440 440 def getDateFromRadarFile(file):
441 441 try:
442 442 year = int(file[1:5])
443 443 doy = int(file[5:8])
444 444 set = int(file[8:11])
445 445 except:
446 446 return None
447 447
448 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 449 return thisDate
450 450
451 451 def getDateFromRadarFolder(folder):
452 452 try:
453 453 year = int(folder[1:5])
454 454 doy = int(folder[5:8])
455 455 except:
456 456 return None
457 457
458 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 459 return thisDate
460 460
461 461 class JRODataIO:
462 462
463 463 c = 3E8
464 464
465 465 isConfig = False
466 466
467 467 basicHeaderObj = None
468 468
469 469 systemHeaderObj = None
470 470
471 471 radarControllerHeaderObj = None
472 472
473 473 processingHeaderObj = None
474 474
475 475 dtype = None
476 476
477 477 pathList = []
478 478
479 479 filenameList = []
480 480
481 481 filename = None
482 482
483 483 ext = None
484 484
485 485 flagIsNewFile = 1
486 486
487 487 flagDiscontinuousBlock = 0
488 488
489 489 flagIsNewBlock = 0
490 490
491 491 fp = None
492 492
493 493 firstHeaderSize = 0
494 494
495 495 basicHeaderSize = 24
496 496
497 497 versionFile = 1103
498 498
499 499 fileSize = None
500 500
501 501 # ippSeconds = None
502 502
503 503 fileSizeByHeader = None
504 504
505 505 fileIndex = None
506 506
507 507 profileIndex = None
508 508
509 509 blockIndex = None
510 510
511 511 nTotalBlocks = None
512 512
513 513 maxTimeStep = 30
514 514
515 515 lastUTTime = None
516 516
517 517 datablock = None
518 518
519 519 dataOut = None
520 520
521 521 blocksize = None
522 522
523 523 getByBlock = False
524 524
525 525 def __init__(self):
526 526
527 527 raise NotImplementedError
528 528
529 529 def run(self):
530 530
531 531 raise NotImplementedError
532 532
533 533 def getDtypeWidth(self):
534 534
535 535 dtype_index = get_dtype_index(self.dtype)
536 536 dtype_width = get_dtype_width(dtype_index)
537 537
538 538 return dtype_width
539 539
540 540 def getAllowedArgs(self):
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544
545 firstTime = True
544
546 545 online = 0
547 546
548 547 realtime = 0
549 548
550 549 nReadBlocks = 0
551 550
552 551 delay = 10 #number of seconds waiting a new file
553 552
554 553 nTries = 3 #quantity tries
555 554
556 555 nFiles = 3 #number of files for searching
557 556
558 557 path = None
559 558
560 559 foldercounter = 0
561 560
562 561 flagNoMoreFiles = 0
563 562
564 563 datetimeList = []
565 564
566 565 __isFirstTimeOnline = 1
567 566
568 567 __printInfo = True
569 568
570 569 profileIndex = None
571 570
572 571 nTxs = 1
573 572
574 573 txIndex = None
575 574
576 575 #Added--------------------
577 576
578 577 selBlocksize = None
579 578
580 579 selBlocktime = None
581
582 onlineWithDate = False
580
583 581 def __init__(self):
584 582
585 583 """
586 584 This class is used to find data files
587 585
588 586 Example:
589 587 reader = JRODataReader()
590 588 fileList = reader.findDataFiles()
591 589
592 590 """
593 591 pass
594 592
595 593
596 594 def createObjByDefault(self):
597 595 """
598 596
599 597 """
600 598 raise NotImplementedError
601 599
602 600 def getBlockDimension(self):
603 601
604 602 raise NotImplementedError
605 603
606 def __searchFilesOffLine(self,
607 path,
608 startDate=None,
609 endDate=None,
610 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
612 set=None,
613 expLabel='',
614 ext='.r',
615 queue=None,
616 cursor=None,
617 skip=None,
618 walk=True):
604 def searchFilesOffLine(self,
605 path,
606 startDate=None,
607 endDate=None,
608 startTime=datetime.time(0,0,0),
609 endTime=datetime.time(23,59,59),
610 set=None,
611 expLabel='',
612 ext='.r',
613 cursor=None,
614 skip=None,
615 walk=True):
616
619 617 self.filenameList = []
620 618 self.datetimeList = []
621 619
622 620 pathList = []
623 621
624 622 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 623
626 624 if dateList == []:
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 return None, None
625 return [], []
629 626
630 627 if len(dateList) > 1:
631 628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
632 629 else:
633 630 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
634 631
635 632 filenameList = []
636 633 datetimeList = []
637 634
638 635 for thisPath in pathList:
639 # thisPath = pathList[pathDict[file]]
640
636
641 637 fileList = glob.glob1(thisPath, "*%s" %ext)
642 638 fileList.sort()
643 639
644 640 skippedFileList = []
645 641
646 if cursor is not None and skip is not None:
647 # if cursor*skip > len(fileList):
642 if cursor is not None andk skip is not None:
643
648 644 if skip == 0:
649 if queue is not None:
650 queue.put(len(fileList))
651 645 skippedFileList = []
652 646 else:
653 647 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
654 648
655 649 else:
656 650 skippedFileList = fileList
657 651
658 652 for file in skippedFileList:
659 653
660 654 filename = os.path.join(thisPath,file)
661 655
662 656 if not isFileInDateRange(filename, startDate, endDate):
663 657 continue
664 658
665 659 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
666 660
667 661 if not(thisDatetime):
668 662 continue
669 663
670 664 filenameList.append(filename)
671 665 datetimeList.append(thisDatetime)
672 666
673 667 if not(filenameList):
674 668 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 return None, None
669 return [], []
676 670
677 671 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 672 print
679 673
680 for i in range(len(filenameList)):
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
674 # for i in range(len(filenameList)):
675 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 676
683 677 self.filenameList = filenameList
684 self.datetimeList = datetimeList
678 self.datetimeList = datetimeList
679
685 680 return pathList, filenameList
686 681
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
688
682 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
683
689 684 """
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 devuelve el archivo encontrado ademas de otros datos.
692
693 Input:
694 path : carpeta donde estan contenidos los files que contiene data
695
696 expLabel : Nombre del subexperimento (subfolder)
697
698 ext : extension de los files
699
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
701
702 Return:
703 directory : eL directorio donde esta el file encontrado
704 filename : el ultimo file de una determinada carpeta
705 year : el anho
706 doy : el numero de dia del anho
707 set : el set del archivo
708
709
685 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
686 devuelve el archivo encontrado ademas de otros datos.
687
688 Input:
689 path : carpeta donde estan contenidos los files que contiene data
690
691 expLabel : Nombre del subexperimento (subfolder)
692
693 ext : extension de los files
694
695 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
696
697 Return:
698 directory : eL directorio donde esta el file encontrado
699 filename : el ultimo file de una determinada carpeta
700 year : el anho
701 doy : el numero de dia del anho
702 set : el set del archivo
703
704
710 705 """
711 pathList = None
712 filenameList = None
713 706 if not os.path.isdir(path):
714 707 return None, None, None, None, None, None
715 708
716 709 dirList = []
717 710
718 711 if not walk:
719 712 fullpath = path
720 713 foldercounter = 0
721 714 else:
722 # Filtra solo los directorios
715 #Filtra solo los directorios
723 716 for thisPath in os.listdir(path):
724 717 if not os.path.isdir(os.path.join(path,thisPath)):
725 718 continue
726 719 if not isRadarFolder(thisPath):
727 720 continue
728 721
729 722 dirList.append(thisPath)
730 723
731 724 if not(dirList):
732 725 return None, None, None, None, None, None
733 726
734 727 dirList = sorted( dirList, key=str.lower )
735 728
736 729 doypath = dirList[-1]
737 730 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 731 fullpath = os.path.join(path, doypath, expLabel)
739 732
740 733
741 734 print "[Reading] %s folder was found: " %(fullpath )
742 735
743 736 if set == None:
744 737 filename = getlastFileFromPath(fullpath, ext)
745 738 else:
746 739 filename = getFileFromSet(fullpath, ext, set)
747 740
748 741 if not(filename):
749 742 return None, None, None, None, None, None
750 743
751 744 print "[Reading] %s file was found" %(filename)
752 745
753 746 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 747 return None, None, None, None, None, None
755 748
756 749 year = int( filename[1:5] )
757 750 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
751 set = int( filename[8:11] )
759 752
760 753 return fullpath, foldercounter, filename, year, doy, set
761 754
762 755 def __setNextFileOffline(self):
763 756
764 757 idFile = self.fileIndex
765 758
766 759 while (True):
767 760 idFile += 1
768 761 if not(idFile < len(self.filenameList)):
769 762 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
763 # print "[Reading] No more Files"
771 764 return 0
772 765
773 766 filename = self.filenameList[idFile]
774 767
775 768 if not(self.__verifyFile(filename)):
776 769 continue
777 770
778 771 fileSize = os.path.getsize(filename)
779 772 fp = open(filename,'rb')
780 773 break
781 774
782 775 self.flagIsNewFile = 1
783 776 self.fileIndex = idFile
784 777 self.filename = filename
785 778 self.fileSize = fileSize
786 779 self.fp = fp
787 780
788 #print "[Reading] Setting the file: %s"%self.filename
781 # print "[Reading] Setting the file: %s"%self.filename
789 782
790 783 return 1
791 784
792 785 def __setNextFileOnline(self):
793 786 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
797
798 Affected:
799 self.flagIsNewFile
800 self.filename
801 self.fileSize
802 self.fp
803 self.set
804 self.flagNoMoreFiles
805
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
810 Excepciones:
811 Si un determinado file no puede ser abierto
812 """
813
787 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
788 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
789 siguientes.
790
791 Affected:
792 self.flagIsNewFile
793 self.filename
794 self.fileSize
795 self.fp
796 self.set
797 self.flagNoMoreFiles
798
799 Return:
800 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
801 1 : si el file fue abierto con exito y esta listo a ser leido
802
803 Excepciones:
804 Si un determinado file no puede ser abierto
805 """
814 806 nFiles = 0
815 807 fileOk_flag = False
816 808 firstTime_flag = True
817 809
818 810 self.set += 1
819 811
820 812 if self.set > 999:
821 813 self.set = 0
822 814 self.foldercounter += 1
823 815
824 816 #busca el 1er file disponible
825 817 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
826 818 if fullfilename:
827 819 if self.__verifyFile(fullfilename, False):
828 820 fileOk_flag = True
829 821
830 822 #si no encuentra un file entonces espera y vuelve a buscar
831 823 if not(fileOk_flag):
832 824 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
833 825
834 826 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
835 827 tries = self.nTries
836 828 else:
837 829 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
838 830
839 831 for nTries in range( tries ):
840 832 if firstTime_flag:
841 833 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
842 834 sleep( self.delay )
843 835 else:
844 836 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
845 837
846 838 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
847 839 if fullfilename:
848 840 if self.__verifyFile(fullfilename):
849 841 fileOk_flag = True
850 842 break
851 843
852 844 if fileOk_flag:
853 845 break
854 846
855 847 firstTime_flag = False
856 848
857 849 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
858 850 self.set += 1
859 851
860 852 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
861 853 self.set = 0
862 854 self.doy += 1
863 855 self.foldercounter = 0
864 856
865 857 if fileOk_flag:
866 858 self.fileSize = os.path.getsize( fullfilename )
867 859 self.filename = fullfilename
868 860 self.flagIsNewFile = 1
869 861 if self.fp != None: self.fp.close()
870 862 self.fp = open(fullfilename, 'rb')
871 863 self.flagNoMoreFiles = 0
872 # print '[Reading] Setting the file: %s' % fullfilename
864 # print '[Reading] Setting the file: %s' % fullfilename
873 865 else:
874 866 self.fileSize = 0
875 867 self.filename = None
876 868 self.flagIsNewFile = 0
877 869 self.fp = None
878 870 self.flagNoMoreFiles = 1
879 # print '[Reading] No more files to read'
871 # print '[Reading] No more files to read'
880 872
881 873 return fileOk_flag
882 874
883 875 def setNextFile(self):
884 876 if self.fp != None:
885 877 self.fp.close()
878
886 879 if self.online:
887 880 newFile = self.__setNextFileOnline()
888 881 else:
889 882 newFile = self.__setNextFileOffline()
883
890 884 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
914 885 print '[Reading] No more files to read'
915 886 return 0
916 887
917 888 if self.verbose:
918 889 print '[Reading] Setting the file: %s' % self.filename
919 890
920 891 self.__readFirstHeader()
921 892 self.nReadBlocks = 0
922 893 return 1
923 894
924 895 def __waitNewBlock(self):
925 896 """
926 897 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
927 898
928 899 Si el modo de lectura es OffLine siempre retorn 0
929 900 """
930 901 if not self.online:
931 902 return 0
932 903
933 904 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
934 905 return 0
935 906
936 907 currentPointer = self.fp.tell()
937 908
938 909 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939 910
940 911 for nTries in range( self.nTries ):
941 912
942 913 self.fp.close()
943 914 self.fp = open( self.filename, 'rb' )
944 915 self.fp.seek( currentPointer )
945 916
946 917 self.fileSize = os.path.getsize( self.filename )
947 918 currentSize = self.fileSize - currentPointer
948 919
949 920 if ( currentSize >= neededSize ):
950 921 self.basicHeaderObj.read(self.fp)
951 922 return 1
952 923
953 924 if self.fileSize == self.fileSizeByHeader:
954 # self.flagEoF = True
925 # self.flagEoF = True
955 926 return 0
956 927
957 928 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
958 929 sleep( self.delay )
959 930
960 931
961 932 return 0
962 933
963 934 def waitDataBlock(self,pointer_location):
964 935
965 936 currentPointer = pointer_location
966 937
967 938 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
968 939
969 940 for nTries in range( self.nTries ):
970 941 self.fp.close()
971 942 self.fp = open( self.filename, 'rb' )
972 943 self.fp.seek( currentPointer )
973 944
974 945 self.fileSize = os.path.getsize( self.filename )
975 946 currentSize = self.fileSize - currentPointer
976 947
977 948 if ( currentSize >= neededSize ):
978 949 return 1
979 950
980 951 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
981 952 sleep( self.delay )
982 953
983 954 return 0
984 955
985 956 def __jumpToLastBlock(self):
986 957
987 958 if not(self.__isFirstTimeOnline):
988 959 return
989 960
990 961 csize = self.fileSize - self.fp.tell()
991 962 blocksize = self.processingHeaderObj.blockSize
992 963
993 964 #salta el primer bloque de datos
994 965 if csize > self.processingHeaderObj.blockSize:
995 966 self.fp.seek(self.fp.tell() + blocksize)
996 967 else:
997 968 return
998 969
999 970 csize = self.fileSize - self.fp.tell()
1000 971 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 972 while True:
1002 973
1003 974 if self.fp.tell()<self.fileSize:
1004 975 self.fp.seek(self.fp.tell() + neededsize)
1005 976 else:
1006 977 self.fp.seek(self.fp.tell() - neededsize)
1007 978 break
1008
1009 # csize = self.fileSize - self.fp.tell()
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 # factor = int(csize/neededsize)
1012 # if factor > 0:
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014
979
980 # csize = self.fileSize - self.fp.tell()
981 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
982 # factor = int(csize/neededsize)
983 # if factor > 0:
984 # self.fp.seek(self.fp.tell() + factor*neededsize)
985
1015 986 self.flagIsNewFile = 0
1016 987 self.__isFirstTimeOnline = 0
1017 988
1018 989 def __setNewBlock(self):
1019 990 #if self.server is None:
1020 991 if self.fp == None:
1021 992 return 0
1022
1023 # if self.online:
1024 # self.__jumpToLastBlock()
1025
993
994 # if self.online:
995 # self.__jumpToLastBlock()
996
1026 997 if self.flagIsNewFile:
1027 998 self.lastUTTime = self.basicHeaderObj.utc
1028 999 return 1
1029 1000
1030 1001 if self.realtime:
1031 1002 self.flagDiscontinuousBlock = 1
1032 1003 if not(self.setNextFile()):
1033 1004 return 0
1034 1005 else:
1035 1006 return 1
1036 1007 #if self.server is None:
1037 1008 currentSize = self.fileSize - self.fp.tell()
1038 1009 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1039 1010 if (currentSize >= neededSize):
1040 1011 self.basicHeaderObj.read(self.fp)
1041 1012 self.lastUTTime = self.basicHeaderObj.utc
1042 1013 return 1
1043 1014 # else:
1044 1015 # self.basicHeaderObj.read(self.zHeader)
1045 1016 # self.lastUTTime = self.basicHeaderObj.utc
1046 1017 # return 1
1047 1018 if self.__waitNewBlock():
1048 1019 self.lastUTTime = self.basicHeaderObj.utc
1049 1020 return 1
1050 1021 #if self.server is None:
1051 1022 if not(self.setNextFile()):
1052 1023 return 0
1053 1024
1054 1025 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1055 1026 self.lastUTTime = self.basicHeaderObj.utc
1056 1027
1057 1028 self.flagDiscontinuousBlock = 0
1058 1029
1059 1030 if deltaTime > self.maxTimeStep:
1060 1031 self.flagDiscontinuousBlock = 1
1061 1032
1062 1033 return 1
1063 1034
1064 1035 def readNextBlock(self):
1065 1036
1066 1037 #Skip block out of startTime and endTime
1067 1038 while True:
1068 if not(self.__setNewBlock()):
1069 print 'returning'
1039 if not(self.__setNewBlock()):
1070 1040 return 0
1041
1071 1042 if not(self.readBlock()):
1072 1043 return 0
1044
1073 1045 self.getBasicHeader()
1046
1074 1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075 1048
1076 1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1077 1050 self.processingHeaderObj.dataBlocksPerFile,
1078 1051 self.dataOut.datatime.ctime())
1079 1052 continue
1080 1053
1081 1054 break
1082 1055
1083 1056 if self.verbose:
1084 1057 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 self.processingHeaderObj.dataBlocksPerFile,
1086 self.dataOut.datatime.ctime())
1058 self.processingHeaderObj.dataBlocksPerFile,
1059 self.dataOut.datatime.ctime())
1087 1060 return 1
1088 1061
1089 1062 def __readFirstHeader(self):
1090 1063
1091 1064 self.basicHeaderObj.read(self.fp)
1092 1065 self.systemHeaderObj.read(self.fp)
1093 1066 self.radarControllerHeaderObj.read(self.fp)
1094 1067 self.processingHeaderObj.read(self.fp)
1095 1068
1096 1069 self.firstHeaderSize = self.basicHeaderObj.size
1097 1070
1098 1071 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1099 1072 if datatype == 0:
1100 1073 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1101 1074 elif datatype == 1:
1102 1075 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1103 1076 elif datatype == 2:
1104 1077 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1105 1078 elif datatype == 3:
1106 1079 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1107 1080 elif datatype == 4:
1108 1081 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1109 1082 elif datatype == 5:
1110 1083 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1111 1084 else:
1112 1085 raise ValueError, 'Data type was not defined'
1113 1086
1114 1087 self.dtype = datatype_str
1115 1088 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 1089 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1090 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1091 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 1092 self.getBlockDimension()
1120 1093
1121 1094 def __verifyFile(self, filename, msgFlag=True):
1122 1095
1123 1096 msg = None
1124 1097
1125 1098 try:
1126 1099 fp = open(filename, 'rb')
1127 1100 except IOError:
1128 1101
1129 1102 if msgFlag:
1130 1103 print "[Reading] File %s can't be opened" % (filename)
1131 1104
1132 1105 return False
1133 1106
1134 1107 currentPosition = fp.tell()
1135 1108 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1136 1109
1137 1110 if neededSize == 0:
1138 1111 basicHeaderObj = BasicHeader(LOCALTIME)
1139 1112 systemHeaderObj = SystemHeader()
1140 1113 radarControllerHeaderObj = RadarControllerHeader()
1141 1114 processingHeaderObj = ProcessingHeader()
1142 1115
1143 1116 if not( basicHeaderObj.read(fp) ):
1144 1117 fp.close()
1145 1118 return False
1146 1119
1147 1120 if not( systemHeaderObj.read(fp) ):
1148 1121 fp.close()
1149 1122 return False
1150 1123
1151 1124 if not( radarControllerHeaderObj.read(fp) ):
1152 1125 fp.close()
1153 1126 return False
1154 1127
1155 1128 if not( processingHeaderObj.read(fp) ):
1156 1129 fp.close()
1157 1130 return False
1158 1131
1159 1132 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1160 1133 else:
1161 1134 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1162 1135
1163 1136 fp.close()
1164 1137
1165 1138 fileSize = os.path.getsize(filename)
1166 1139 currentSize = fileSize - currentPosition
1167 1140
1168 1141 if currentSize < neededSize:
1169 1142 if msgFlag and (msg != None):
1170 1143 print msg
1171 1144 return False
1172 1145
1173 1146 return True
1174 1147
1175 1148 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1176 1149
1177 1150 path_empty = True
1178 1151
1179 1152 dateList = []
1180 1153 pathList = []
1181 1154
1182 1155 multi_path = path.split(',')
1183 1156
1184 1157 if not walk:
1185 1158
1186 1159 for single_path in multi_path:
1187 1160
1188 1161 if not os.path.isdir(single_path):
1189 1162 continue
1190 1163
1191 1164 fileList = glob.glob1(single_path, "*"+ext)
1192 1165
1193 1166 if not fileList:
1194 1167 continue
1195 1168
1196 1169 path_empty = False
1197 1170
1198 1171 fileList.sort()
1199 1172
1200 1173 for thisFile in fileList:
1201 1174
1202 1175 if not os.path.isfile(os.path.join(single_path, thisFile)):
1203 1176 continue
1204 1177
1205 1178 if not isRadarFile(thisFile):
1206 1179 continue
1207 1180
1208 1181 if not isFileInDateRange(thisFile, startDate, endDate):
1209 1182 continue
1210 1183
1211 1184 thisDate = getDateFromRadarFile(thisFile)
1212 1185
1213 1186 if thisDate in dateList:
1214 1187 continue
1215 1188
1216 1189 dateList.append(thisDate)
1217 1190 pathList.append(single_path)
1218 1191
1219 1192 else:
1220 1193 for single_path in multi_path:
1221 1194
1222 1195 if not os.path.isdir(single_path):
1223 1196 continue
1224 1197
1225 1198 dirList = []
1226 1199
1227 1200 for thisPath in os.listdir(single_path):
1228 1201
1229 1202 if not os.path.isdir(os.path.join(single_path,thisPath)):
1230 1203 continue
1231 1204
1232 1205 if not isRadarFolder(thisPath):
1233 1206 continue
1234 1207
1235 1208 if not isFolderInDateRange(thisPath, startDate, endDate):
1236 1209 continue
1237 1210
1238 1211 dirList.append(thisPath)
1239 1212
1240 1213 if not dirList:
1241 1214 continue
1242 1215
1243 1216 dirList.sort()
1244 1217
1245 1218 for thisDir in dirList:
1246 1219
1247 1220 datapath = os.path.join(single_path, thisDir, expLabel)
1248 1221 fileList = glob.glob1(datapath, "*"+ext)
1249 1222
1250 1223 if not fileList:
1251 1224 continue
1252 1225
1253 1226 path_empty = False
1254 1227
1255 1228 thisDate = getDateFromRadarFolder(thisDir)
1256 1229
1257 1230 pathList.append(datapath)
1258 1231 dateList.append(thisDate)
1259 1232
1260 1233 dateList.sort()
1261 1234
1262 1235 if walk:
1263 1236 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1264 1237 else:
1265 1238 pattern_path = multi_path[0]
1266 1239
1267 1240 if path_empty:
1268 1241 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1269 1242 else:
1270 1243 if not dateList:
1271 1244 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1272 1245
1273 1246 if include_path:
1274 1247 return dateList, pathList
1275 1248
1276 1249 return dateList
1277 1250
1278 1251 def setup(self,
1279 1252 path=None,
1280 startDate=None,
1281 endDate=None,
1282 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1284 set=None,
1285 expLabel = "",
1286 ext = None,
1253 startDate=None,
1254 endDate=None,
1255 startTime=datetime.time(0,0,0),
1256 endTime=datetime.time(23,59,59),
1257 set=None,
1258 expLabel = "",
1259 ext = None,
1287 1260 online = False,
1288 1261 delay = 60,
1289 1262 walk = True,
1290 1263 getblock = False,
1291 1264 nTxs = 1,
1292 1265 realtime=False,
1293 1266 blocksize=None,
1294 1267 blocktime=None,
1268 skip=None,
1269 cursor=None,
1270 warnings=True,
1295 1271 verbose=True,
1272 server=None,
1296 1273 **kwargs):
1297
1298 if path == None:
1299 raise ValueError, "[Reading] The path is not valid"
1300
1274 if server is not None:
1275 if 'tcp://' in server:
1276 address = server
1277 else:
1278 address = 'ipc:///tmp/%s' % server
1279 self.server = address
1280 self.context = zmq.Context()
1281 self.receiver = self.context.socket(zmq.PULL)
1282 self.receiver.connect(self.server)
1283 time.sleep(0.5)
1284 print '[Starting] ReceiverData from {}'.format(self.server)
1285 else:
1286 self.server = None
1287 if path == None:
1288 raise ValueError, "[Reading] The path is not valid"
1301 1289
1302 if ext == None:
1303 ext = self.ext
1304
1305 self.verbose=verbose
1306 self.path = path
1307 self.startDate = startDate
1308 self.endDate = endDate
1309 self.startTime = startTime
1310 self.endTime = endTime
1311 self.set = set
1312 self.expLabel = expLabel
1313 self.ext = ext
1314 self.online = online
1315 self.delay = delay
1316 self.walk = walk
1317 self.getblock = getblock
1318 self.nTxs = nTxs
1319 self.realtime = realtime
1320 self.blocksize = blocksize
1321 self.blocktime = blocktime
1322
1323
1324 if self.firstTime is True:
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1290 if ext == None:
1291 ext = self.ext
1292
1293 if online:
1294 print "[Reading] Searching files in online mode..."
1295
1296 for nTries in range( self.nTries ):
1297 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1298
1299 if fullpath:
1300 break
1301
1302 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1303 sleep( self.delay )
1304
1305 if not(fullpath):
1306 print "[Reading] There 'isn't any valid file in %s" % path
1307 return
1308
1309 self.year = year
1310 self.doy = doy
1311 self.set = set - 1
1312 self.path = path
1313 self.foldercounter = foldercounter
1314 last_set = None
1315 else:
1316 print "[Reading] Searching files in offline mode ..."
1317 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 1318 startTime=startTime, endTime=endTime,
1327 1319 set=set, expLabel=expLabel, ext=ext,
1328 walk=walk)
1329 if filenameList is not None: filenameList = filenameList[:-1]
1320 walk=walk, cursor=cursor,
1321 skip=skip)
1330 1322
1331 if pathList is not None and filenameList is not None and online:
1332 self.onlineWithDate = True
1333 online = False
1323 if not(pathList):
1334 1324 self.fileIndex = -1
1335 self.pathList = pathList
1336 self.filenameList = filenameList
1337 file_name = os.path.basename(filenameList[-1])
1338 basename, ext = os.path.splitext(file_name)
1339 last_set = int(basename[-3:])
1340
1341 if online:
1342 print "[Reading] Searching files in online mode..."
1343
1344 for nTries in range(self.nTries):
1345 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1346 expLabel=expLabel,
1347 ext=ext,
1348 walk=walk,
1349 startDate=startDate,
1350 startTime=startTime,
1351 set=set)
1352
1353 if fullpath:
1354 break
1355 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1356 sleep( self.delay )
1357
1358 if not(fullpath):
1359 print "[Reading] There 'isn't any valid file in %s" % path
1360 return
1361
1362 self.year = year
1363 self.doy = doy
1364 self.set = set - 1
1365 self.path = path
1366 self.foldercounter = foldercounter
1367 last_set = None
1368 else:
1369 print "[Reading] Searching files in offline mode ..."
1370 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1371 startTime=startTime, endTime=endTime,
1372 set=set, expLabel=expLabel, ext=ext,
1373 walk=walk)
1374
1375 if not(pathList):
1376 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1377 # datetime.datetime.combine(startDate,startTime).ctime(),
1378 # datetime.datetime.combine(endDate,endTime).ctime())
1379
1380 # sys.exit(-1)
1381
1382 self.fileIndex = -1
1383 self.pathList = []
1384 self.filenameList = []
1385 return
1386
1387 self.fileIndex = -1
1388 self.pathList = pathList
1389 self.filenameList = filenameList
1390 file_name = os.path.basename(filenameList[-1])
1391 basename, ext = os.path.splitext(file_name)
1392 last_set = int(basename[-3:])
1393
1394
1395 self.online = online
1396 self.realtime = realtime
1397 self.delay = delay
1398 ext = ext.lower()
1399 self.ext = ext
1400 self.getByBlock = getblock
1401 self.nTxs = nTxs
1402 self.startTime = startTime
1403 self.endTime = endTime
1404
1405
1406 #Added-----------------
1407 self.selBlocksize = blocksize
1408 self.selBlocktime = blocktime
1409
1410
1411 if not(self.setNextFile()):
1412 if (startDate!=None) and (endDate!=None):
1413 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1414 elif startDate != None:
1415 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1416 else:
1417 print "[Reading] No files"
1418
1325 self.pathList = []
1326 self.filenameList = []
1327 return
1328
1419 1329 self.fileIndex = -1
1420 self.pathList = []
1421 self.filenameList = []
1422 return
1330 self.pathList = pathList
1331 self.filenameList = filenameList
1332 file_name = os.path.basename(filenameList[-1])
1333 basename, ext = os.path.splitext(file_name)
1334 last_set = int(basename[-3:])
1335
1336 self.online = online
1337 self.realtime = realtime
1338 self.delay = delay
1339 ext = ext.lower()
1340 self.ext = ext
1341 self.getByBlock = getblock
1342 self.nTxs = nTxs
1343 self.startTime = startTime
1344 self.endTime = endTime
1345
1346 #Added-----------------
1347 self.selBlocksize = blocksize
1348 self.selBlocktime = blocktime
1349
1350 # Verbose-----------
1351 self.verbose = verbose
1352 self.warnings = warnings
1353
1354 if not(self.setNextFile()):
1355 if (startDate!=None) and (endDate!=None):
1356 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1357 elif startDate != None:
1358 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1359 else:
1360 print "[Reading] No files"
1423 1361
1424 # self.getBasicHeader()
1362 self.fileIndex = -1
1363 self.pathList = []
1364 self.filenameList = []
1365 return
1425 1366
1426 if last_set != None:
1427 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1367 # self.getBasicHeader()
1368
1369 if last_set != None:
1370 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1428 1371 return
1429 1372
1430 1373 def getBasicHeader(self):
1431 1374
1432 1375 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1433 1376
1434 1377 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1435 1378
1436 1379 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1437 1380
1438 1381 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1439 1382
1440 1383 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1441 1384
1442 1385 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1443 1386
1444 1387 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1445
1446 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1447
1448
1388
1389 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1390
1391
1449 1392 def getFirstHeader(self):
1450 1393
1451 1394 raise NotImplementedError
1452 1395
1453 1396 def getData(self):
1454 1397
1455 1398 raise NotImplementedError
1456 1399
1457 1400 def hasNotDataInBuffer(self):
1458 1401
1459 1402 raise NotImplementedError
1460 1403
1461 1404 def readBlock(self):
1462 1405
1463 1406 raise NotImplementedError
1464 1407
1465 1408 def isEndProcess(self):
1466 1409
1467 1410 return self.flagNoMoreFiles
1468 1411
1469 1412 def printReadBlocks(self):
1470 1413
1471 1414 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1472 1415
1473 1416 def printTotalBlocks(self):
1474 1417
1475 1418 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1476 1419
1477 1420 def printNumberOfBlock(self):
1478 1421
1479 1422 if self.flagIsNewBlock:
1480 1423 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1481 1424 self.processingHeaderObj.dataBlocksPerFile,
1482 1425 self.dataOut.datatime.ctime())
1483 1426
1484 1427 def printInfo(self):
1485 1428
1486 1429 if self.__printInfo == False:
1487 1430 return
1488 1431
1489 1432 self.basicHeaderObj.printInfo()
1490 1433 self.systemHeaderObj.printInfo()
1491 1434 self.radarControllerHeaderObj.printInfo()
1492 1435 self.processingHeaderObj.printInfo()
1493 1436
1494 1437 self.__printInfo = False
1495 1438
1496 1439 def run(self,
1497 1440 path=None,
1498 1441 startDate=None,
1499 1442 endDate=None,
1500 1443 startTime=datetime.time(0,0,0),
1501 1444 endTime=datetime.time(23,59,59),
1502 1445 set=None,
1503 1446 expLabel = "",
1504 1447 ext = None,
1505 1448 online = False,
1506 1449 delay = 60,
1507 1450 walk = True,
1508 1451 getblock = False,
1509 1452 nTxs = 1,
1510 1453 realtime=False,
1511 1454 blocksize=None,
1512 1455 blocktime=None,
1513 1456 queue=None,
1514 1457 skip=None,
1515 1458 cursor=None,
1516 1459 warnings=True,
1517 1460 server=None,
1518 1461 verbose=True, **kwargs):
1519 1462 if not(self.isConfig):
1520 # self.dataOut = dataOut
1521 self.setup( path=path,
1522 startDate=startDate,
1523 endDate=endDate,
1524 startTime=startTime,
1525 endTime=endTime,
1526 set=set,
1527 expLabel=expLabel,
1528 ext=ext,
1529 online=online,
1530 delay=delay,
1531 walk=walk,
1532 getblock=getblock,
1533 nTxs=nTxs,
1534 realtime=realtime,
1535 blocksize=blocksize,
1536 blocktime=blocktime,
1537 queue=queue,
1538 skip=skip,
1539 cursor=cursor,
1540 warnings=warnings,
1541 server=server,
1542 verbose=verbose, **kwargs)
1463 self.setup(path=path,
1464 startDate=startDate,
1465 endDate=endDate,
1466 startTime=startTime,
1467 endTime=endTime,
1468 set=set,
1469 expLabel=expLabel,
1470 ext=ext,
1471 online=online,
1472 delay=delay,
1473 walk=walk,
1474 getblock=getblock,
1475 nTxs=nTxs,
1476 realtime=realtime,
1477 blocksize=blocksize,
1478 blocktime=blocktime,
1479 skip=skip,
1480 cursor=cursor,
1481 warnings=warnings,
1482 server=server,
1483 verbose=verbose)
1543 1484 self.isConfig = True
1544 1485 if server is None:
1545 1486 self.getData()
1546 1487 else:
1547 1488 self.getFromServer()
1548 1489
1549 1490 class JRODataWriter(JRODataIO):
1550 1491
1551 1492 """
1552 1493 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1553 1494 de los datos siempre se realiza por bloques.
1554 1495 """
1555 1496
1556 1497 blockIndex = 0
1557 1498
1558 1499 path = None
1559 1500
1560 1501 setFile = None
1561 1502
1562 1503 profilesPerBlock = None
1563 1504
1564 1505 blocksPerFile = None
1565 1506
1566 1507 nWriteBlocks = 0
1567 1508
1568 1509 fileDate = None
1569 1510
1570 1511 def __init__(self, dataOut=None):
1571 1512 raise NotImplementedError
1572 1513
1573 1514
1574 1515 def hasAllDataInBuffer(self):
1575 1516 raise NotImplementedError
1576 1517
1577 1518
1578 1519 def setBlockDimension(self):
1579 1520 raise NotImplementedError
1580 1521
1581 1522
1582 1523 def writeBlock(self):
1583 1524 raise NotImplementedError
1584 1525
1585 1526
1586 1527 def putData(self):
1587 1528 raise NotImplementedError
1588 1529
1589 1530
1590 1531 def getProcessFlags(self):
1591 1532
1592 1533 processFlags = 0
1593 1534
1594 1535 dtype_index = get_dtype_index(self.dtype)
1595 1536 procflag_dtype = get_procflag_dtype(dtype_index)
1596 1537
1597 1538 processFlags += procflag_dtype
1598 1539
1599 1540 if self.dataOut.flagDecodeData:
1600 1541 processFlags += PROCFLAG.DECODE_DATA
1601 1542
1602 1543 if self.dataOut.flagDeflipData:
1603 1544 processFlags += PROCFLAG.DEFLIP_DATA
1604 1545
1605 1546 if self.dataOut.code is not None:
1606 1547 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1607 1548
1608 1549 if self.dataOut.nCohInt > 1:
1609 1550 processFlags += PROCFLAG.COHERENT_INTEGRATION
1610 1551
1611 1552 if self.dataOut.type == "Spectra":
1612 1553 if self.dataOut.nIncohInt > 1:
1613 1554 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1614 1555
1615 1556 if self.dataOut.data_dc is not None:
1616 1557 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1617 1558
1618 1559 if self.dataOut.flagShiftFFT:
1619 1560 processFlags += PROCFLAG.SHIFT_FFT_DATA
1620 1561
1621 1562 return processFlags
1622 1563
1623 1564 def setBasicHeader(self):
1624 1565
1625 1566 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1626 1567 self.basicHeaderObj.version = self.versionFile
1627 1568 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1628 1569
1629 1570 utc = numpy.floor(self.dataOut.utctime)
1630 1571 milisecond = (self.dataOut.utctime - utc)* 1000.0
1631 1572
1632 1573 self.basicHeaderObj.utc = utc
1633 1574 self.basicHeaderObj.miliSecond = milisecond
1634 1575 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1635 1576 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1636 1577 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1637 1578
1638 1579 def setFirstHeader(self):
1639 1580 """
1640 1581 Obtiene una copia del First Header
1641 1582
1642 1583 Affected:
1643 1584
1644 1585 self.basicHeaderObj
1645 1586 self.systemHeaderObj
1646 1587 self.radarControllerHeaderObj
1647 1588 self.processingHeaderObj self.
1648 1589
1649 1590 Return:
1650 1591 None
1651 1592 """
1652 1593
1653 1594 raise NotImplementedError
1654 1595
1655 1596 def __writeFirstHeader(self):
1656 1597 """
1657 1598 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1658 1599
1659 1600 Affected:
1660 1601 __dataType
1661 1602
1662 1603 Return:
1663 1604 None
1664 1605 """
1665
1666 # CALCULAR PARAMETROS
1667
1606
1607 # CALCULAR PARAMETROS
1608
1668 1609 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1669 1610 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1670 1611
1671 1612 self.basicHeaderObj.write(self.fp)
1672 1613 self.systemHeaderObj.write(self.fp)
1673 1614 self.radarControllerHeaderObj.write(self.fp)
1674 1615 self.processingHeaderObj.write(self.fp)
1675 1616
1676 1617 def __setNewBlock(self):
1677 1618 """
1678 1619 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1679 1620
1680 1621 Return:
1681 1622 0 : si no pudo escribir nada
1682 1623 1 : Si escribio el Basic el First Header
1683 1624 """
1684 1625 if self.fp == None:
1685 1626 self.setNextFile()
1686 1627
1687 1628 if self.flagIsNewFile:
1688 1629 return 1
1689 1630
1690 1631 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1691 1632 self.basicHeaderObj.write(self.fp)
1692 1633 return 1
1693 1634
1694 1635 if not( self.setNextFile() ):
1695 1636 return 0
1696 1637
1697 1638 return 1
1698 1639
1699 1640
1700 1641 def writeNextBlock(self):
1701 1642 """
1702 1643 Selecciona el bloque siguiente de datos y los escribe en un file
1703 1644
1704 1645 Return:
1705 1646 0 : Si no hizo pudo escribir el bloque de datos
1706 1647 1 : Si no pudo escribir el bloque de datos
1707 1648 """
1708 1649 if not( self.__setNewBlock() ):
1709 1650 return 0
1710 1651
1711 1652 self.writeBlock()
1712 1653
1713 1654 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1714 1655 self.processingHeaderObj.dataBlocksPerFile)
1715 1656
1716 1657 return 1
1717 1658
1718 1659 def setNextFile(self):
1719 1660 """
1720 1661 Determina el siguiente file que sera escrito
1721 1662
1722 1663 Affected:
1723 1664 self.filename
1724 1665 self.subfolder
1725 1666 self.fp
1726 1667 self.setFile
1727 1668 self.flagIsNewFile
1728 1669
1729 1670 Return:
1730 1671 0 : Si el archivo no puede ser escrito
1731 1672 1 : Si el archivo esta listo para ser escrito
1732 1673 """
1733 1674 ext = self.ext
1734 1675 path = self.path
1735 1676
1736 1677 if self.fp != None:
1737 1678 self.fp.close()
1738 1679
1739 1680 timeTuple = time.localtime( self.dataOut.utctime)
1740 1681 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1741 1682
1742 1683 fullpath = os.path.join( path, subfolder )
1743 1684 setFile = self.setFile
1744 1685
1745 1686 if not( os.path.exists(fullpath) ):
1746 1687 os.mkdir(fullpath)
1747 1688 setFile = -1 #inicializo mi contador de seteo
1748 1689 else:
1749 1690 filesList = os.listdir( fullpath )
1750 1691 if len( filesList ) > 0:
1751 1692 filesList = sorted( filesList, key=str.lower )
1752 1693 filen = filesList[-1]
1753 1694 # el filename debera tener el siguiente formato
1754 1695 # 0 1234 567 89A BCDE (hex)
1755 1696 # x YYYY DDD SSS .ext
1756 1697 if isNumber( filen[8:11] ):
1757 1698 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1758 1699 else:
1759 1700 setFile = -1
1760 1701 else:
1761 1702 setFile = -1 #inicializo mi contador de seteo
1762 1703
1763 1704 setFile += 1
1764 1705
1765 1706 #If this is a new day it resets some values
1766 1707 if self.dataOut.datatime.date() > self.fileDate:
1767 1708 setFile = 0
1768 1709 self.nTotalBlocks = 0
1769 1710
1770 1711 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1771 1712
1772 1713 filename = os.path.join( path, subfolder, filen )
1773 1714
1774 1715 fp = open( filename,'wb' )
1775 1716
1776 1717 self.blockIndex = 0
1777 1718
1778 1719 #guardando atributos
1779 1720 self.filename = filename
1780 1721 self.subfolder = subfolder
1781 1722 self.fp = fp
1782 1723 self.setFile = setFile
1783 1724 self.flagIsNewFile = 1
1784 1725 self.fileDate = self.dataOut.datatime.date()
1785 1726
1786 1727 self.setFirstHeader()
1787 1728
1788 1729 print '[Writing] Opening file: %s'%self.filename
1789 1730
1790 1731 self.__writeFirstHeader()
1791 1732
1792 1733 return 1
1793 1734
1794 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True):
1735 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1795 1736 """
1796 1737 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1797 1738
1798 1739 Inputs:
1799 1740 path : directory where data will be saved
1800 1741 profilesPerBlock : number of profiles per block
1801 1742 set : initial file set
1802 1743 datatype : An integer number that defines data type:
1803 1744 0 : int8 (1 byte)
1804 1745 1 : int16 (2 bytes)
1805 1746 2 : int32 (4 bytes)
1806 1747 3 : int64 (8 bytes)
1807 1748 4 : float32 (4 bytes)
1808 1749 5 : double64 (8 bytes)
1809 1750
1810 1751 Return:
1811 1752 0 : Si no realizo un buen seteo
1812 1753 1 : Si realizo un buen seteo
1813 1754 """
1814 1755
1815 1756 if ext == None:
1816 1757 ext = self.ext
1817 1758
1818 1759 self.ext = ext.lower()
1819 1760
1820 1761 self.path = path
1821 1762
1822 1763 if set is None:
1823 1764 self.setFile = -1
1824 1765 else:
1825 1766 self.setFile = set - 1
1826 1767
1827 1768 self.blocksPerFile = blocksPerFile
1828 1769
1829 1770 self.profilesPerBlock = profilesPerBlock
1830 1771
1831 1772 self.dataOut = dataOut
1832 1773 self.fileDate = self.dataOut.datatime.date()
1833 1774 #By default
1834 1775 self.dtype = self.dataOut.dtype
1835 1776
1836 1777 if datatype is not None:
1837 1778 self.dtype = get_numpy_dtype(datatype)
1838 1779
1839 1780 if not(self.setNextFile()):
1840 1781 print "[Writing] There isn't a next file"
1841 1782 return 0
1842 1783
1843 1784 self.setBlockDimension()
1844 1785
1845 1786 return 1
1846 1787
1847 1788 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1848 1789
1849 1790 if not(self.isConfig):
1850 1791
1851 1792 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1852 1793 self.isConfig = True
1853 1794
1854 1795 self.putData()
Show file before | Show file after | Hide comments
@@ -1,848 +1,848
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import os, sys
8 8 import time, datetime
9 9 import numpy
10 10 import fnmatch
11 11 import glob
12 12 from time import sleep
13 13
14 14 try:
15 15 import pyfits
16 16 except ImportError, e:
17 17 print "Fits data cannot be used. Install pyfits module"
18 18
19 19 from xml.etree.ElementTree import ElementTree
20 20
21 21 from jroIO_base import isRadarFolder, isNumber
22 22 from schainpy.model.data.jrodata import Fits
23 23 from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit
24 24
25 25 class PyFits(object):
26 26 name=None
27 27 format=None
28 28 array =None
29 29 data =None
30 30 thdulist=None
31 31 prihdr=None
32 32 hdu=None
33 33
34 34 def __init__(self):
35 35
36 36 pass
37 37
38 38 def setColF(self,name,format,array):
39 39 self.name=name
40 40 self.format=format
41 41 self.array=array
42 42 a1=numpy.array([self.array],dtype=numpy.float32)
43 43 self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
44 44 return self.col1
45 45
46 46 # def setColP(self,name,format,data):
47 47 # self.name=name
48 48 # self.format=format
49 49 # self.data=data
50 50 # a2=numpy.array([self.data],dtype=numpy.float32)
51 51 # self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
52 52 # return self.col2
53 53
54 54
55 55 def writeData(self,name,format,data):
56 56 self.name=name
57 57 self.format=format
58 58 self.data=data
59 59 a2=numpy.array([self.data],dtype=numpy.float32)
60 60 self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
61 61 return self.col2
62 62
63 63 def cFImage(self,idblock,year,month,day,hour,minute,second):
64 64 self.hdu= pyfits.PrimaryHDU(idblock)
65 65 self.hdu.header.set("Year",year)
66 66 self.hdu.header.set("Month",month)
67 67 self.hdu.header.set("Day",day)
68 68 self.hdu.header.set("Hour",hour)
69 69 self.hdu.header.set("Minute",minute)
70 70 self.hdu.header.set("Second",second)
71 71 return self.hdu
72 72
73 73
74 74 def Ctable(self,colList):
75 75 self.cols=pyfits.ColDefs(colList)
76 76 self.tbhdu = pyfits.new_table(self.cols)
77 77 return self.tbhdu
78 78
79 79
80 80 def CFile(self,hdu,tbhdu):
81 81 self.thdulist=pyfits.HDUList([hdu,tbhdu])
82 82
83 83 def wFile(self,filename):
84 84 if os.path.isfile(filename):
85 85 os.remove(filename)
86 86 self.thdulist.writeto(filename)
87 87
88 88
89 89 class ParameterConf:
90 90 ELEMENTNAME = 'Parameter'
91 91 def __init__(self):
92 92 self.name = ''
93 93 self.value = ''
94 94
95 95 def readXml(self, parmElement):
96 96 self.name = parmElement.get('name')
97 97 self.value = parmElement.get('value')
98 98
99 99 def getElementName(self):
100 100 return self.ELEMENTNAME
101 101
102 102 class Metadata(object):
103 103
104 104 def __init__(self, filename):
105 105 self.parmConfObjList = []
106 106 self.readXml(filename)
107 107
108 108 def readXml(self, filename):
109 109 self.projectElement = None
110 110 self.procUnitConfObjDict = {}
111 111 self.projectElement = ElementTree().parse(filename)
112 112 self.project = self.projectElement.tag
113 113
114 114 parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
115 115
116 116 for parmElement in parmElementList:
117 117 parmConfObj = ParameterConf()
118 118 parmConfObj.readXml(parmElement)
119 119 self.parmConfObjList.append(parmConfObj)
120 120
121 121 class FitsWriter(Operation):
122 122 def __init__(self, **kwargs):
123 123 Operation.__init__(self, **kwargs)
124 124 self.isConfig = False
125 125 self.dataBlocksPerFile = None
126 126 self.blockIndex = 0
127 127 self.flagIsNewFile = 1
128 128 self.fitsObj = None
129 129 self.optchar = 'P'
130 130 self.ext = '.fits'
131 131 self.setFile = 0
132 132
133 133 def setFitsHeader(self, dataOut, metadatafile=None):
134 134
135 135 header_data = pyfits.PrimaryHDU()
136 136
137 137 header_data.header['EXPNAME'] = "RADAR DATA"
138 138 header_data.header['DATATYPE'] = "SPECTRA"
139 139 header_data.header['COMMENT'] = ""
140 140
141 141 if metadatafile:
142 142
143 143 metadata4fits = Metadata(metadatafile)
144 144
145 145 for parameter in metadata4fits.parmConfObjList:
146 146 parm_name = parameter.name
147 147 parm_value = parameter.value
148 148
149 149 header_data.header[parm_name] = parm_value
150 150
151 151 header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
152 152 header_data.header['CHANNELLIST'] = str(dataOut.channelList)
153 153 header_data.header['NCHANNELS'] = dataOut.nChannels
154 154 #header_data.header['HEIGHTS'] = dataOut.heightList
155 155 header_data.header['NHEIGHTS'] = dataOut.nHeights
156 156
157 157 header_data.header['IPPSECONDS'] = dataOut.ippSeconds
158 158 header_data.header['NCOHINT'] = dataOut.nCohInt
159 159 header_data.header['NINCOHINT'] = dataOut.nIncohInt
160 160 header_data.header['TIMEZONE'] = dataOut.timeZone
161 161 header_data.header['NBLOCK'] = self.blockIndex
162 162
163 163 header_data.writeto(self.filename)
164 164
165 165 self.addExtension(dataOut.heightList,'HEIGHTLIST')
166 166
167 167
168 168 def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None):
169 169
170 170 self.path = path
171 171 self.dataOut = dataOut
172 172 self.metadatafile = metadatafile
173 173 self.dataBlocksPerFile = dataBlocksPerFile
174 174
175 175 def open(self):
176 176 self.fitsObj = pyfits.open(self.filename, mode='update')
177 177
178 178
179 179 def addExtension(self, data, tagname):
180 180 self.open()
181 181 extension = pyfits.ImageHDU(data=data, name=tagname)
182 182 #extension.header['TAG'] = tagname
183 183 self.fitsObj.append(extension)
184 184 self.write()
185 185
186 186 def addData(self, data):
187 187 self.open()
188 188 extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
189 189 extension.header['UTCTIME'] = self.dataOut.utctime
190 190 self.fitsObj.append(extension)
191 191 self.blockIndex += 1
192 192 self.fitsObj[0].header['NBLOCK'] = self.blockIndex
193 193
194 194 self.write()
195 195
196 196 def write(self):
197 197
198 198 self.fitsObj.flush(verbose=True)
199 199 self.fitsObj.close()
200 200
201 201
202 202 def setNextFile(self):
203 203
204 204 ext = self.ext
205 205 path = self.path
206 206
207 207 timeTuple = time.localtime( self.dataOut.utctime)
208 208 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
209 209
210 210 fullpath = os.path.join( path, subfolder )
211 211 if not( os.path.exists(fullpath) ):
212 212 os.mkdir(fullpath)
213 213 self.setFile = -1 #inicializo mi contador de seteo
214 214 else:
215 215 filesList = os.listdir( fullpath )
216 216 if len( filesList ) > 0:
217 217 filesList = sorted( filesList, key=str.lower )
218 218 filen = filesList[-1]
219 219
220 220 if isNumber( filen[8:11] ):
221 221 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
222 222 else:
223 223 self.setFile = -1
224 224 else:
225 225 self.setFile = -1 #inicializo mi contador de seteo
226 226
227 227 setFile = self.setFile
228 228 setFile += 1
229 229
230 230 thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
231 231 timeTuple.tm_year,
232 232 timeTuple.tm_yday,
233 233 setFile,
234 234 ext )
235 235
236 236 filename = os.path.join( path, subfolder, thisFile )
237 237
238 238 self.blockIndex = 0
239 239 self.filename = filename
240 240 self.setFile = setFile
241 241 self.flagIsNewFile = 1
242 242
243 243 print 'Writing the file: %s'%self.filename
244 244
245 245 self.setFitsHeader(self.dataOut, self.metadatafile)
246 246
247 247 return 1
248 248
249 249 def writeBlock(self):
250 250 self.addData(self.dataOut.data_spc)
251 251 self.flagIsNewFile = 0
252 252
253 253
254 254 def __setNewBlock(self):
255 255
256 256 if self.flagIsNewFile:
257 257 return 1
258 258
259 259 if self.blockIndex < self.dataBlocksPerFile:
260 260 return 1
261 261
262 262 if not( self.setNextFile() ):
263 263 return 0
264 264
265 265 return 1
266 266
267 267 def writeNextBlock(self):
268 268 if not( self.__setNewBlock() ):
269 269 return 0
270 270 self.writeBlock()
271 271 return 1
272 272
273 273 def putData(self):
274 274 if self.flagIsNewFile:
275 275 self.setNextFile()
276 276 self.writeNextBlock()
277 277
278 278 def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs):
279 279 if not(self.isConfig):
280 280 self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs)
281 281 self.isConfig = True
282 282 self.putData()
283 283
284 284
285 285 class FitsReader(ProcessingUnit):
286 286
287 287 # __TIMEZONE = time.timezone
288 288
289 289 expName = None
290 290 datetimestr = None
291 291 utc = None
292 292 nChannels = None
293 293 nSamples = None
294 294 dataBlocksPerFile = None
295 295 comments = None
296 296 lastUTTime = None
297 297 header_dict = None
298 298 data = None
299 299 data_header_dict = None
300 300
301 301 def __init__(self, **kwargs):
302 302 ProcessingUnit.__init__(self, **kwargs)
303 303 self.isConfig = False
304 304 self.ext = '.fits'
305 305 self.setFile = 0
306 306 self.flagNoMoreFiles = 0
307 307 self.flagIsNewFile = 1
308 308 self.flagDiscontinuousBlock = None
309 309 self.fileIndex = None
310 310 self.filename = None
311 311 self.fileSize = None
312 312 self.fitsObj = None
313 313 self.timeZone = None
314 314 self.nReadBlocks = 0
315 315 self.nTotalBlocks = 0
316 316 self.dataOut = self.createObjByDefault()
317 317 self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
318 318 self.blockIndex = 1
319 319
320 320 def createObjByDefault(self):
321 321
322 322 dataObj = Fits()
323 323
324 324 return dataObj
325 325
326 326 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
327 327 try:
328 328 fitsObj = pyfits.open(filename,'readonly')
329 329 except:
330 330 print "File %s can't be opened" %(filename)
331 331 return None
332 332
333 333 header = fitsObj[0].header
334 334 struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
335 335 utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
336 336
337 337 ltc = utc
338 338 if useLocalTime:
339 339 ltc -= time.timezone
340 340 thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
341 341 thisTime = thisDatetime.time()
342 342
343 343 if not ((startTime <= thisTime) and (endTime > thisTime)):
344 344 return None
345 345
346 346 return thisDatetime
347 347
348 348 def __setNextFileOnline(self):
349 349 raise NotImplementedError
350 350
351 351 def __setNextFileOffline(self):
352 352 idFile = self.fileIndex
353 353
354 354 while (True):
355 355 idFile += 1
356 356 if not(idFile < len(self.filenameList)):
357 357 self.flagNoMoreFiles = 1
358 358 print "No more Files"
359 359 return 0
360 360
361 361 filename = self.filenameList[idFile]
362 362
363 363 # if not(self.__verifyFile(filename)):
364 364 # continue
365 365
366 366 fileSize = os.path.getsize(filename)
367 367 fitsObj = pyfits.open(filename,'readonly')
368 368 break
369 369
370 370 self.flagIsNewFile = 1
371 371 self.fileIndex = idFile
372 372 self.filename = filename
373 373 self.fileSize = fileSize
374 374 self.fitsObj = fitsObj
375 375 self.blockIndex = 0
376 376 print "Setting the file: %s"%self.filename
377 377
378 378 return 1
379 379
380 380 def __setValuesFromHeader(self):
381 381
382 382 self.dataOut.header = self.header_dict
383 383 self.dataOut.expName = self.expName
384 384
385 385 self.dataOut.timeZone = self.timeZone
386 386 self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
387 387 self.dataOut.comments = self.comments
388 388 # self.dataOut.timeInterval = self.timeInterval
389 389 self.dataOut.channelList = self.channelList
390 390 self.dataOut.heightList = self.heightList
391 391
392 392 self.dataOut.nCohInt = self.nCohInt
393 393 self.dataOut.nIncohInt = self.nIncohInt
394 394
395 395 self.dataOut.ippSeconds = self.ippSeconds
396 396
397 397 def readHeader(self):
398 398 headerObj = self.fitsObj[0]
399 399
400 400 self.header_dict = headerObj.header
401 401 if 'EXPNAME' in headerObj.header.keys():
402 402 self.expName = headerObj.header['EXPNAME']
403 403
404 404 if 'DATATYPE' in headerObj.header.keys():
405 405 self.dataType = headerObj.header['DATATYPE']
406 406
407 407 self.datetimestr = headerObj.header['DATETIME']
408 408 channelList = headerObj.header['CHANNELLIST']
409 409 channelList = channelList.split('[')
410 410 channelList = channelList[1].split(']')
411 411 channelList = channelList[0].split(',')
412 412 channelList = [int(ch) for ch in channelList]
413 413 self.channelList = channelList
414 414 self.nChannels = headerObj.header['NCHANNELS']
415 415 self.nHeights = headerObj.header['NHEIGHTS']
416 416 self.ippSeconds = headerObj.header['IPPSECONDS']
417 417 self.nCohInt = headerObj.header['NCOHINT']
418 418 self.nIncohInt = headerObj.header['NINCOHINT']
419 419 self.dataBlocksPerFile = headerObj.header['NBLOCK']
420 420 self.timeZone = headerObj.header['TIMEZONE']
421 421
422 422 # self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
423 423
424 424 if 'COMMENT' in headerObj.header.keys():
425 425 self.comments = headerObj.header['COMMENT']
426 426
427 427 self.readHeightList()
428 428
429 429 def readHeightList(self):
430 430 self.blockIndex = self.blockIndex + 1
431 431 obj = self.fitsObj[self.blockIndex]
432 432 self.heightList = obj.data
433 433 self.blockIndex = self.blockIndex + 1
434 434
435 435 def readExtension(self):
436 436 obj = self.fitsObj[self.blockIndex]
437 437 self.heightList = obj.data
438 438 self.blockIndex = self.blockIndex + 1
439 439
440 440 def setNextFile(self):
441 441
442 442 if self.online:
443 443 newFile = self.__setNextFileOnline()
444 444 else:
445 445 newFile = self.__setNextFileOffline()
446 446
447 447 if not(newFile):
448 448 return 0
449 449
450 450 self.readHeader()
451 451 self.__setValuesFromHeader()
452 452 self.nReadBlocks = 0
453 453 # self.blockIndex = 1
454 454 return 1
455 455
456 def __searchFilesOffLine(self,
456 def searchFilesOffLine(self,
457 457 path,
458 458 startDate,
459 459 endDate,
460 460 startTime=datetime.time(0,0,0),
461 461 endTime=datetime.time(23,59,59),
462 462 set=None,
463 463 expLabel='',
464 464 ext='.fits',
465 465 walk=True):
466 466
467 467 pathList = []
468 468
469 469 if not walk:
470 470 pathList.append(path)
471 471
472 472 else:
473 473 dirList = []
474 474 for thisPath in os.listdir(path):
475 475 if not os.path.isdir(os.path.join(path,thisPath)):
476 476 continue
477 477 if not isRadarFolder(thisPath):
478 478 continue
479 479
480 480 dirList.append(thisPath)
481 481
482 482 if not(dirList):
483 483 return None, None
484 484
485 485 thisDate = startDate
486 486
487 487 while(thisDate <= endDate):
488 488 year = thisDate.timetuple().tm_year
489 489 doy = thisDate.timetuple().tm_yday
490 490
491 491 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
492 492 if len(matchlist) == 0:
493 493 thisDate += datetime.timedelta(1)
494 494 continue
495 495 for match in matchlist:
496 496 pathList.append(os.path.join(path,match,expLabel))
497 497
498 498 thisDate += datetime.timedelta(1)
499 499
500 500 if pathList == []:
501 501 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
502 502 return None, None
503 503
504 504 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
505 505
506 506 filenameList = []
507 507 datetimeList = []
508 508
509 509 for i in range(len(pathList)):
510 510
511 511 thisPath = pathList[i]
512 512
513 513 fileList = glob.glob1(thisPath, "*%s" %ext)
514 514 fileList.sort()
515 515
516 516 for thisFile in fileList:
517 517
518 518 filename = os.path.join(thisPath,thisFile)
519 519 thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
520 520
521 521 if not(thisDatetime):
522 522 continue
523 523
524 524 filenameList.append(filename)
525 525 datetimeList.append(thisDatetime)
526 526
527 527 if not(filenameList):
528 528 print "Any file was found for the time range %s - %s" %(startTime, endTime)
529 529 return None, None
530 530
531 531 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
532 532 print
533 533
534 534 for i in range(len(filenameList)):
535 535 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
536 536
537 537 self.filenameList = filenameList
538 538 self.datetimeList = datetimeList
539 539
540 540 return pathList, filenameList
541 541
542 542 def setup(self, path=None,
543 543 startDate=None,
544 544 endDate=None,
545 545 startTime=datetime.time(0,0,0),
546 546 endTime=datetime.time(23,59,59),
547 547 set=0,
548 548 expLabel = "",
549 549 ext = None,
550 550 online = False,
551 551 delay = 60,
552 552 walk = True):
553 553
554 554 if path == None:
555 555 raise ValueError, "The path is not valid"
556 556
557 557 if ext == None:
558 558 ext = self.ext
559 559
560 560 if not(online):
561 561 print "Searching files in offline mode ..."
562 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
562 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
563 563 startTime=startTime, endTime=endTime,
564 564 set=set, expLabel=expLabel, ext=ext,
565 565 walk=walk)
566 566
567 567 if not(pathList):
568 568 print "No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
569 569 datetime.datetime.combine(startDate,startTime).ctime(),
570 570 datetime.datetime.combine(endDate,endTime).ctime())
571 571
572 572 sys.exit(-1)
573 573
574 574 self.fileIndex = -1
575 575 self.pathList = pathList
576 576 self.filenameList = filenameList
577 577
578 578 self.online = online
579 579 self.delay = delay
580 580 ext = ext.lower()
581 581 self.ext = ext
582 582
583 583 if not(self.setNextFile()):
584 584 if (startDate!=None) and (endDate!=None):
585 585 print "No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
586 586 elif startDate != None:
587 587 print "No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
588 588 else:
589 589 print "No files"
590 590
591 591 sys.exit(-1)
592 592
593 593
594 594
595 595 def readBlock(self):
596 596 dataObj = self.fitsObj[self.blockIndex]
597 597
598 598 self.data = dataObj.data
599 599 self.data_header_dict = dataObj.header
600 600 self.utc = self.data_header_dict['UTCTIME']
601 601
602 602 self.flagIsNewFile = 0
603 603 self.blockIndex += 1
604 604 self.nTotalBlocks += 1
605 605 self.nReadBlocks += 1
606 606
607 607 return 1
608 608
609 609 def __jumpToLastBlock(self):
610 610 raise NotImplementedError
611 611
612 612 def __waitNewBlock(self):
613 613 """
614 614 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
615 615
616 616 Si el modo de lectura es OffLine siempre retorn 0
617 617 """
618 618 if not self.online:
619 619 return 0
620 620
621 621 if (self.nReadBlocks >= self.dataBlocksPerFile):
622 622 return 0
623 623
624 624 currentPointer = self.fp.tell()
625 625
626 626 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
627 627
628 628 for nTries in range( self.nTries ):
629 629
630 630 self.fp.close()
631 631 self.fp = open( self.filename, 'rb' )
632 632 self.fp.seek( currentPointer )
633 633
634 634 self.fileSize = os.path.getsize( self.filename )
635 635 currentSize = self.fileSize - currentPointer
636 636
637 637 if ( currentSize >= neededSize ):
638 638 self.__rdBasicHeader()
639 639 return 1
640 640
641 641 print "\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
642 642 sleep( self.delay )
643 643
644 644
645 645 return 0
646 646
647 647 def __setNewBlock(self):
648 648
649 649 if self.online:
650 650 self.__jumpToLastBlock()
651 651
652 652 if self.flagIsNewFile:
653 653 return 1
654 654
655 655 self.lastUTTime = self.utc
656 656
657 657 if self.online:
658 658 if self.__waitNewBlock():
659 659 return 1
660 660
661 661 if self.nReadBlocks < self.dataBlocksPerFile:
662 662 return 1
663 663
664 664 if not(self.setNextFile()):
665 665 return 0
666 666
667 667 deltaTime = self.utc - self.lastUTTime
668 668
669 669 self.flagDiscontinuousBlock = 0
670 670
671 671 if deltaTime > self.maxTimeStep:
672 672 self.flagDiscontinuousBlock = 1
673 673
674 674 return 1
675 675
676 676
677 677 def readNextBlock(self):
678 678 if not(self.__setNewBlock()):
679 679 return 0
680 680
681 681 if not(self.readBlock()):
682 682 return 0
683 683
684 684 return 1
685 685
686 686 def printInfo(self):
687 687
688 688 pass
689 689
690 690 def getData(self):
691 691
692 692 if self.flagNoMoreFiles:
693 693 self.dataOut.flagNoData = True
694 694 print 'Process finished'
695 695 return 0
696 696
697 697 self.flagDiscontinuousBlock = 0
698 698 self.flagIsNewBlock = 0
699 699
700 700 if not(self.readNextBlock()):
701 701 return 0
702 702
703 703 if self.data is None:
704 704 self.dataOut.flagNoData = True
705 705 return 0
706 706
707 707 self.dataOut.data = self.data
708 708 self.dataOut.data_header = self.data_header_dict
709 709 self.dataOut.utctime = self.utc
710 710
711 711 # self.dataOut.header = self.header_dict
712 712 # self.dataOut.expName = self.expName
713 713 # self.dataOut.nChannels = self.nChannels
714 714 # self.dataOut.timeZone = self.timeZone
715 715 # self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
716 716 # self.dataOut.comments = self.comments
717 717 # # self.dataOut.timeInterval = self.timeInterval
718 718 # self.dataOut.channelList = self.channelList
719 719 # self.dataOut.heightList = self.heightList
720 720 self.dataOut.flagNoData = False
721 721
722 722 return self.dataOut.data
723 723
724 724 def run(self, **kwargs):
725 725
726 726 if not(self.isConfig):
727 727 self.setup(**kwargs)
728 728 self.isConfig = True
729 729
730 730 self.getData()
731 731
732 732 class SpectraHeisWriter(Operation):
733 733 # set = None
734 734 setFile = None
735 735 idblock = None
736 736 doypath = None
737 737 subfolder = None
738 738
739 739 def __init__(self, **kwargs):
740 740 Operation.__init__(self, **kwargs)
741 741 self.wrObj = PyFits()
742 742 # self.dataOut = dataOut
743 743 self.nTotalBlocks=0
744 744 # self.set = None
745 745 self.setFile = None
746 746 self.idblock = 0
747 747 self.wrpath = None
748 748 self.doypath = None
749 749 self.subfolder = None
750 750 self.isConfig = False
751 751
752 752 def isNumber(str):
753 753 """
754 754 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
755 755
756 756 Excepciones:
757 757 Si un determinado string no puede ser convertido a numero
758 758 Input:
759 759 str, string al cual se le analiza para determinar si convertible a un numero o no
760 760
761 761 Return:
762 762 True : si el string es uno numerico
763 763 False : no es un string numerico
764 764 """
765 765 try:
766 766 float( str )
767 767 return True
768 768 except:
769 769 return False
770 770
771 771 def setup(self, dataOut, wrpath):
772 772
773 773 if not(os.path.exists(wrpath)):
774 774 os.mkdir(wrpath)
775 775
776 776 self.wrpath = wrpath
777 777 # self.setFile = 0
778 778 self.dataOut = dataOut
779 779
780 780 def putData(self):
781 781 name= time.localtime( self.dataOut.utctime)
782 782 ext=".fits"
783 783
784 784 if self.doypath == None:
785 785 self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
786 786 self.doypath = os.path.join( self.wrpath, self.subfolder )
787 787 os.mkdir(self.doypath)
788 788
789 789 if self.setFile == None:
790 790 # self.set = self.dataOut.set
791 791 self.setFile = 0
792 792 # if self.set != self.dataOut.set:
793 793 ## self.set = self.dataOut.set
794 794 # self.setFile = 0
795 795
796 796 #make the filename
797 797 thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
798 798
799 799 filename = os.path.join(self.wrpath,self.subfolder, thisFile)
800 800
801 801 idblock = numpy.array([self.idblock],dtype="int64")
802 802 header=self.wrObj.cFImage(idblock=idblock,
803 803 year=time.gmtime(self.dataOut.utctime).tm_year,
804 804 month=time.gmtime(self.dataOut.utctime).tm_mon,
805 805 day=time.gmtime(self.dataOut.utctime).tm_mday,
806 806 hour=time.gmtime(self.dataOut.utctime).tm_hour,
807 807 minute=time.gmtime(self.dataOut.utctime).tm_min,
808 808 second=time.gmtime(self.dataOut.utctime).tm_sec)
809 809
810 810 c=3E8
811 811 deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
812 812 freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
813 813
814 814 colList = []
815 815
816 816 colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
817 817
818 818 colList.append(colFreq)
819 819
820 820 nchannel=self.dataOut.nChannels
821 821
822 822 for i in range(nchannel):
823 823 col = self.wrObj.writeData(name="PCh"+str(i+1),
824 824 format=str(self.dataOut.nFFTPoints)+'E',
825 825 data=10*numpy.log10(self.dataOut.data_spc[i,:]))
826 826
827 827 colList.append(col)
828 828
829 829 data=self.wrObj.Ctable(colList=colList)
830 830
831 831 self.wrObj.CFile(header,data)
832 832
833 833 self.wrObj.wFile(filename)
834 834
835 835 #update the setFile
836 836 self.setFile += 1
837 837 self.idblock += 1
838 838
839 839 return 1
840 840
841 841 def run(self, dataOut, **kwargs):
842 842
843 843 if not(self.isConfig):
844 844
845 845 self.setup(dataOut, **kwargs)
846 846 self.isConfig = True
847 847
848 848 self.putData()
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
The requested commit or file is too big and content was truncated. Show full diff
General Comments 0
You need to be logged in to leave comments. Login now