##// END OF EJS Templates
schain
RSS
formatting
José Chávez -
r1082:831a23834344
parent child
Show More
Side by Side Unified
Context file:
r1082:831a23834344 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1297 +1,1313
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 12 from multiprocessing import Process, cpu_count
13 13
14 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 15 from xml.dom import minidom
16 16
17 17 import schainpy
18 18 import schainpy.admin
19 19 from schainpy.model import *
20 20 from schainpy.utils import log
21 21
22 22 DTYPES = {
23 23 'Voltage': '.r',
24 24 'Spectra': '.pdata'
25 25 }
26 26
27
27 28 def MPProject(project, n=cpu_count()):
28 29 '''
29 30 Project wrapper to run schain in n processes
30 31 '''
31 32
32 33 rconf = project.getReadUnitObj()
33 34 op = rconf.getOperationObj('run')
34 35 dt1 = op.getParameterValue('startDate')
35 36 dt2 = op.getParameterValue('endDate')
36 37 days = (dt2 - dt1).days
37
38 for day in range(days+1):
38
39 for day in range(days + 1):
39 40 skip = 0
40 41 cursor = 0
41 42 processes = []
42 43 dt = dt1 + datetime.timedelta(day)
43 44 dt_str = dt.strftime('%Y/%m/%d')
44 45 reader = JRODataReader()
45 46 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 startDate=dt,
47 endDate=dt,
48 ext=DTYPES[rconf.datatype])
47 startDate=dt,
48 endDate=dt,
49 ext=DTYPES[rconf.datatype])
49 50 nFiles = len(files)
50 51 if nFiles == 0:
51 52 continue
52 skip = int(math.ceil(nFiles/n))
53 while nFiles > cursor*skip:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 skip=skip)
56 p = project.clone()
53 skip = int(math.ceil(nFiles / n))
54 while nFiles > cursor * skip:
55 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
56 skip=skip)
57 p = project.clone()
57 58 p.start()
58 59 processes.append(p)
59 60 cursor += 1
60 61
61 62 def beforeExit(exctype, value, trace):
62 63 for process in processes:
63 64 process.terminate()
64 65 process.join()
65 66 print traceback.print_tb(trace)
66 67
67 68 sys.excepthook = beforeExit
68 69
69 70 for process in processes:
70 71 process.join()
71 72 process.terminate()
72 73
73 74 time.sleep(3)
74 75
76
75 77 class ParameterConf():
76 78
77 79 id = None
78 80 name = None
79 81 value = None
80 82 format = None
81 83
82 84 __formated_value = None
83 85
84 86 ELEMENTNAME = 'Parameter'
85 87
86 88 def __init__(self):
87 89
88 90 self.format = 'str'
89 91
90 92 def getElementName(self):
91 93
92 94 return self.ELEMENTNAME
93 95
94 96 def getValue(self):
95 97
96 98 value = self.value
97 99 format = self.format
98 100
99 101 if self.__formated_value != None:
100 102
101 103 return self.__formated_value
102 104
103 105 if format == 'obj':
104 106 return value
105 107
106 108 if format == 'str':
107 109 self.__formated_value = str(value)
108 110 return self.__formated_value
109 111
110 112 if value == '':
111 raise ValueError, '%s: This parameter value is empty' %self.name
113 raise ValueError, '%s: This parameter value is empty' % self.name
112 114
113 115 if format == 'list':
114 116 strList = value.split(',')
115 117
116 118 self.__formated_value = strList
117 119
118 120 return self.__formated_value
119 121
120 122 if format == 'intlist':
121 123 '''
122 124 Example:
123 125 value = (0,1,2)
124 126 '''
125 127
126 128 new_value = ast.literal_eval(value)
127 129
128 130 if type(new_value) not in (tuple, list):
129 131 new_value = [int(new_value)]
130 132
131 133 self.__formated_value = new_value
132 134
133 135 return self.__formated_value
134 136
135 137 if format == 'floatlist':
136 138 '''
137 139 Example:
138 140 value = (0.5, 1.4, 2.7)
139 141 '''
140 142
141 143 new_value = ast.literal_eval(value)
142 144
143 145 if type(new_value) not in (tuple, list):
144 146 new_value = [float(new_value)]
145 147
146 148 self.__formated_value = new_value
147 149
148 150 return self.__formated_value
149 151
150 152 if format == 'date':
151 153 strList = value.split('/')
152 154 intList = [int(x) for x in strList]
153 155 date = datetime.date(intList[0], intList[1], intList[2])
154 156
155 157 self.__formated_value = date
156 158
157 159 return self.__formated_value
158 160
159 161 if format == 'time':
160 162 strList = value.split(':')
161 163 intList = [int(x) for x in strList]
162 164 time = datetime.time(intList[0], intList[1], intList[2])
163 165
164 166 self.__formated_value = time
165 167
166 168 return self.__formated_value
167 169
168 170 if format == 'pairslist':
169 171 '''
170 172 Example:
171 173 value = (0,1),(1,2)
172 174 '''
173 175
174 176 new_value = ast.literal_eval(value)
175 177
176 178 if type(new_value) not in (tuple, list):
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
179 raise ValueError, '%s has to be a tuple or list of pairs' % value
178 180
179 181 if type(new_value[0]) not in (tuple, list):
180 182 if len(new_value) != 2:
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
182 184 new_value = [new_value]
183 185
184 186 for thisPair in new_value:
185 187 if len(thisPair) != 2:
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
188 raise ValueError, '%s has to be a tuple or list of pairs' % value
187 189
188 190 self.__formated_value = new_value
189 191
190 192 return self.__formated_value
191 193
192 194 if format == 'multilist':
193 195 '''
194 196 Example:
195 197 value = (0,1,2),(3,4,5)
196 198 '''
197 199 multiList = ast.literal_eval(value)
198 200
199 201 if type(multiList[0]) == int:
200 202 multiList = ast.literal_eval('(' + value + ')')
201 203
202 204 self.__formated_value = multiList
203 205
204 206 return self.__formated_value
205 207
206 208 if format == 'bool':
207 209 value = int(value)
208 210
209 211 if format == 'int':
210 212 value = float(value)
211 213
212 214 format_func = eval(format)
213 215
214 216 self.__formated_value = format_func(value)
215 217
216 218 return self.__formated_value
217 219
218 220 def updateId(self, new_id):
219 221
220 222 self.id = str(new_id)
221 223
222 224 def setup(self, id, name, value, format='str'):
223 225 self.id = str(id)
224 226 self.name = name
225 227 if format == 'obj':
226 228 self.value = value
227 229 else:
228 230 self.value = str(value)
229 231 self.format = str.lower(format)
230 232
231 233 self.getValue()
232 234
233 235 return 1
234 236
235 237 def update(self, name, value, format='str'):
236 238
237 239 self.name = name
238 240 self.value = str(value)
239 241 self.format = format
240 242
241 243 def makeXml(self, opElement):
242 244 if self.name not in ('queue',):
243 245 parmElement = SubElement(opElement, self.ELEMENTNAME)
244 246 parmElement.set('id', str(self.id))
245 247 parmElement.set('name', self.name)
246 248 parmElement.set('value', self.value)
247 249 parmElement.set('format', self.format)
248 250
249 251 def readXml(self, parmElement):
250 252
251 253 self.id = parmElement.get('id')
252 254 self.name = parmElement.get('name')
253 255 self.value = parmElement.get('value')
254 256 self.format = str.lower(parmElement.get('format'))
255 257
256 #Compatible with old signal chain version
258 # Compatible with old signal chain version
257 259 if self.format == 'int' and self.name == 'idfigure':
258 260 self.name = 'id'
259 261
260 262 def printattr(self):
261 263
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
264 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
265
263 266
264 267 class OperationConf():
265 268
266 269 id = None
267 270 name = None
268 271 priority = None
269 272 type = None
270 273
271 274 parmConfObjList = []
272 275
273 276 ELEMENTNAME = 'Operation'
274 277
275 278 def __init__(self):
276 279
277 280 self.id = '0'
278 281 self.name = None
279 282 self.priority = None
280 283 self.type = 'self'
281 284
282
283 285 def __getNewId(self):
284 286
285 return int(self.id)*10 + len(self.parmConfObjList) + 1
287 return int(self.id) * 10 + len(self.parmConfObjList) + 1
286 288
287 289 def updateId(self, new_id):
288 290
289 291 self.id = str(new_id)
290 292
291 293 n = 1
292 294 for parmObj in self.parmConfObjList:
293 295
294 idParm = str(int(new_id)*10 + n)
296 idParm = str(int(new_id) * 10 + n)
295 297 parmObj.updateId(idParm)
296 298
297 299 n += 1
298 300
299 301 def getElementName(self):
300 302
301 303 return self.ELEMENTNAME
302 304
303 305 def getParameterObjList(self):
304 306
305 307 return self.parmConfObjList
306 308
307 309 def getParameterObj(self, parameterName):
308 310
309 311 for parmConfObj in self.parmConfObjList:
310 312
311 313 if parmConfObj.name != parameterName:
312 314 continue
313 315
314 316 return parmConfObj
315 317
316 318 return None
317 319
318 320 def getParameterObjfromValue(self, parameterValue):
319 321
320 322 for parmConfObj in self.parmConfObjList:
321 323
322 324 if parmConfObj.getValue() != parameterValue:
323 325 continue
324 326
325 327 return parmConfObj.getValue()
326 328
327 329 return None
328 330
329 331 def getParameterValue(self, parameterName):
330 332
331 333 parameterObj = self.getParameterObj(parameterName)
332
334
333 335 # if not parameterObj:
334 336 # return None
335
337
336 338 value = parameterObj.getValue()
337 339
338 340 return value
339 341
340
341 342 def getKwargs(self):
342 343
343 344 kwargs = {}
344 345
345 346 for parmConfObj in self.parmConfObjList:
346 347 if self.name == 'run' and parmConfObj.name == 'datatype':
347 348 continue
348 349
349 350 kwargs[parmConfObj.name] = parmConfObj.getValue()
350 351
351 352 return kwargs
352 353
353 354 def setup(self, id, name, priority, type):
354 355
355 356 self.id = str(id)
356 357 self.name = name
357 358 self.type = type
358 359 self.priority = priority
359 360
360 361 self.parmConfObjList = []
361 362
362 363 def removeParameters(self):
363 364
364 365 for obj in self.parmConfObjList:
365 366 del obj
366 367
367 368 self.parmConfObjList = []
368 369
369 370 def addParameter(self, name, value, format='str'):
370
371
371 372 if value is None:
372 373 return None
373 374 id = self.__getNewId()
374 375
375 376 parmConfObj = ParameterConf()
376 377 if not parmConfObj.setup(id, name, value, format):
377 378 return None
378 379
379 380 self.parmConfObjList.append(parmConfObj)
380 381
381 382 return parmConfObj
382 383
383 384 def changeParameter(self, name, value, format='str'):
384 385
385 386 parmConfObj = self.getParameterObj(name)
386 387 parmConfObj.update(name, value, format)
387 388
388 389 return parmConfObj
389 390
390 391 def makeXml(self, procUnitElement):
391 392
392 393 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
393 394 opElement.set('id', str(self.id))
394 395 opElement.set('name', self.name)
395 396 opElement.set('type', self.type)
396 397 opElement.set('priority', str(self.priority))
397 398
398 399 for parmConfObj in self.parmConfObjList:
399 400 parmConfObj.makeXml(opElement)
400 401
401 402 def readXml(self, opElement):
402 403
403 404 self.id = opElement.get('id')
404 405 self.name = opElement.get('name')
405 406 self.type = opElement.get('type')
406 407 self.priority = opElement.get('priority')
407 408
408 #Compatible with old signal chain version
409 #Use of 'run' method instead 'init'
409 # Compatible with old signal chain version
410 # Use of 'run' method instead 'init'
410 411 if self.type == 'self' and self.name == 'init':
411 412 self.name = 'run'
412 413
413 414 self.parmConfObjList = []
414 415
415 416 parmElementList = opElement.iter(ParameterConf().getElementName())
416 417
417 418 for parmElement in parmElementList:
418 419 parmConfObj = ParameterConf()
419 420 parmConfObj.readXml(parmElement)
420 421
421 #Compatible with old signal chain version
422 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
422 # Compatible with old signal chain version
423 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
423 424 if self.type != 'self' and self.name == 'Plot':
424 425 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
425 426 self.name = parmConfObj.value
426 427 continue
427 428
428 429 self.parmConfObjList.append(parmConfObj)
429 430
430 431 def printattr(self):
431 432
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
433 self.id,
434 self.name,
435 self.type,
436 self.priority)
433 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
434 self.id,
435 self.name,
436 self.type,
437 self.priority)
437 438
438 439 for parmConfObj in self.parmConfObjList:
439 440 parmConfObj.printattr()
440 441
441 442 def createObject(self, plotter_queue=None):
442 443
443
444 444 if self.type == 'self':
445 445 raise ValueError, 'This operation type cannot be created'
446 446
447 if self.type == 'plotter':
447 if self.type == 'plotter':
448 448 if not plotter_queue:
449 449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
450 450
451 451 opObj = Plotter(self.name, plotter_queue)
452 452
453 453 if self.type == 'external' or self.type == 'other':
454 454
455 455 className = eval(self.name)
456 456 kwargs = self.getKwargs()
457 457
458 458 opObj = className(**kwargs)
459 459
460 460 return opObj
461 461
462 462
463 463 class ProcUnitConf():
464 464
465 465 id = None
466 466 name = None
467 467 datatype = None
468 468 inputId = None
469 469 parentId = None
470 470
471 471 opConfObjList = []
472 472
473 473 procUnitObj = None
474 474 opObjList = []
475 475
476 476 ELEMENTNAME = 'ProcUnit'
477 477
478 478 def __init__(self):
479 479
480 480 self.id = None
481 481 self.datatype = None
482 482 self.name = None
483 483 self.inputId = None
484 484
485 485 self.opConfObjList = []
486 486
487 487 self.procUnitObj = None
488 488 self.opObjDict = {}
489 489
490 490 def __getPriority(self):
491 491
492 return len(self.opConfObjList)+1
492 return len(self.opConfObjList) + 1
493 493
494 494 def __getNewId(self):
495 495
496 return int(self.id)*10 + len(self.opConfObjList) + 1
496 return int(self.id) * 10 + len(self.opConfObjList) + 1
497 497
498 498 def getElementName(self):
499 499
500 500 return self.ELEMENTNAME
501 501
502 502 def getId(self):
503 503
504 504 return self.id
505 505
506 506 def updateId(self, new_id, parentId=parentId):
507 507
508 new_id = int(parentId) * 10 + (int(self.id) % 10)
509 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
508 510
509 new_id = int(parentId)*10 + (int(self.id) % 10)
510 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
511
512 #If this proc unit has not inputs
511 # If this proc unit has not inputs
513 512 if self.inputId == '0':
514 513 new_inputId = 0
515 514
516 515 n = 1
517 516 for opConfObj in self.opConfObjList:
518 517
519 idOp = str(int(new_id)*10 + n)
518 idOp = str(int(new_id) * 10 + n)
520 519 opConfObj.updateId(idOp)
521 520
522 521 n += 1
523 522
524 523 self.parentId = str(parentId)
525 524 self.id = str(new_id)
526 525 self.inputId = str(new_inputId)
527 526
528
529 527 def getInputId(self):
530 528
531 529 return self.inputId
532 530
533 531 def getOperationObjList(self):
534 532
535 533 return self.opConfObjList
536 534
537 535 def getOperationObj(self, name=None):
538 536
539 537 for opConfObj in self.opConfObjList:
540 538
541 539 if opConfObj.name != name:
542 540 continue
543 541
544 542 return opConfObj
545 543
546 544 return None
547 545
548 546 def getOpObjfromParamValue(self, value=None):
549 547
550 548 for opConfObj in self.opConfObjList:
551 549 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
552 550 continue
553 551 return opConfObj
554 552 return None
555 553
556 554 def getProcUnitObj(self):
557 555
558 556 return self.procUnitObj
559 557
560 558 def setup(self, id, name, datatype, inputId, parentId=None):
561 559
562 #Compatible with old signal chain version
563 if datatype==None and name==None:
560 # Compatible with old signal chain version
561 if datatype == None and name == None:
564 562 raise ValueError, 'datatype or name should be defined'
565 563
566 if name==None:
564 if name == None:
567 565 if 'Proc' in datatype:
568 566 name = datatype
569 567 else:
570 name = '%sProc' %(datatype)
568 name = '%sProc' % (datatype)
571 569
572 if datatype==None:
573 datatype = name.replace('Proc','')
570 if datatype == None:
571 datatype = name.replace('Proc', '')
574 572
575 573 self.id = str(id)
576 574 self.name = name
577 575 self.datatype = datatype
578 576 self.inputId = inputId
579 577 self.parentId = parentId
580 578
581 579 self.opConfObjList = []
582 580
583 581 self.addOperation(name='run', optype='self')
584 582
585 583 def removeOperations(self):
586 584
587 585 for obj in self.opConfObjList:
588 586 del obj
589 587
590 588 self.opConfObjList = []
591 589 self.addOperation(name='run')
592 590
593 591 def addParameter(self, **kwargs):
594 592 '''
595 593 Add parameters to 'run' operation
596 594 '''
597 595 opObj = self.opConfObjList[0]
598 596
599 597 opObj.addParameter(**kwargs)
600 598
601 599 return opObj
602 600
603 601 def addOperation(self, name, optype='self'):
604 602
605 603 id = self.__getNewId()
606 604 priority = self.__getPriority()
607 605
608 606 opConfObj = OperationConf()
609 607 opConfObj.setup(id, name=name, priority=priority, type=optype)
610 608
611 609 self.opConfObjList.append(opConfObj)
612 610
613 611 return opConfObj
614 612
615 613 def makeXml(self, projectElement):
616 614
617 615 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
618 616 procUnitElement.set('id', str(self.id))
619 617 procUnitElement.set('name', self.name)
620 618 procUnitElement.set('datatype', self.datatype)
621 619 procUnitElement.set('inputId', str(self.inputId))
622 620
623 621 for opConfObj in self.opConfObjList:
624 622 opConfObj.makeXml(procUnitElement)
625 623
626 624 def readXml(self, upElement):
627 625
628 626 self.id = upElement.get('id')
629 627 self.name = upElement.get('name')
630 628 self.datatype = upElement.get('datatype')
631 629 self.inputId = upElement.get('inputId')
632 630
633 631 if self.ELEMENTNAME == 'ReadUnit':
634 632 self.datatype = self.datatype.replace('Reader', '')
635 633
636 634 if self.ELEMENTNAME == 'ProcUnit':
637 635 self.datatype = self.datatype.replace('Proc', '')
638 636
639 637 if self.inputId == 'None':
640 638 self.inputId = '0'
641 639
642 640 self.opConfObjList = []
643 641
644 642 opElementList = upElement.iter(OperationConf().getElementName())
645 643
646 644 for opElement in opElementList:
647 645 opConfObj = OperationConf()
648 646 opConfObj.readXml(opElement)
649 647 self.opConfObjList.append(opConfObj)
650 648
651 649 def printattr(self):
652 650
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
654 self.id,
655 self.name,
656 self.datatype,
657 self.inputId)
658
651 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
652 self.id,
653 self.name,
654 self.datatype,
655 self.inputId)
656
659 657 for opConfObj in self.opConfObjList:
660 658 opConfObj.printattr()
661 659
662
663 660 def getKwargs(self):
664 661
665 662 opObj = self.opConfObjList[0]
666 663 kwargs = opObj.getKwargs()
667 664
668 665 return kwargs
669 666
670 667 def createObjects(self, plotter_queue=None):
671 668
672 669 className = eval(self.name)
673 670 kwargs = self.getKwargs()
674 671 procUnitObj = className(**kwargs)
675 672
676 673 for opConfObj in self.opConfObjList:
677 674
678 if opConfObj.type=='self' and self.name=='run':
675 if opConfObj.type == 'self' and self.name == 'run':
679 676 continue
680 elif opConfObj.type=='self':
681 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
677 elif opConfObj.type == 'self':
678 procUnitObj.addOperationKwargs(
679 opConfObj.id, **opConfObj.getKwargs())
682 680 continue
683 681
684 682 opObj = opConfObj.createObject(plotter_queue)
685 683
686 684 self.opObjDict[opConfObj.id] = opObj
687 685
688 686 procUnitObj.addOperation(opObj, opConfObj.id)
689 687
690 688 self.procUnitObj = procUnitObj
691 689
692 690 return procUnitObj
693 691
694 692 def run(self):
695 693
696 694 is_ok = False
697 695
698 696 for opConfObj in self.opConfObjList:
699 697
700 698 kwargs = {}
701 699 for parmConfObj in opConfObj.getParameterObjList():
702 700 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
703 701 continue
704 702
705 703 kwargs[parmConfObj.name] = parmConfObj.getValue()
706 704
707 sts = self.procUnitObj.call(opType = opConfObj.type,
708 opName = opConfObj.name,
709 opId = opConfObj.id)
710
705 sts = self.procUnitObj.call(opType=opConfObj.type,
706 opName=opConfObj.name,
707 opId=opConfObj.id)
708
711 709 is_ok = is_ok or sts
712 710
713 711 return is_ok
714 712
715 713 def close(self):
716 714
717 715 for opConfObj in self.opConfObjList:
718 716 if opConfObj.type == 'self':
719 717 continue
720 718
721 719 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
722 720 opObj.close()
723 721
724 722 self.procUnitObj.close()
725 723
726 724 return
727 725
726
728 727 class ReadUnitConf(ProcUnitConf):
729 728
730 729 path = None
731 730 startDate = None
732 731 endDate = None
733 732 startTime = None
734 733 endTime = None
735 734
736 735 ELEMENTNAME = 'ReadUnit'
737 736
738 737 def __init__(self):
739 738
740 739 self.id = None
741 740 self.datatype = None
742 741 self.name = None
743 742 self.inputId = None
744 743
745 744 self.parentId = None
746 745
747 746 self.opConfObjList = []
748 747 self.opObjList = []
749 748
750 749 def getElementName(self):
751 750
752 751 return self.ELEMENTNAME
753 752
754 753 def setup(self, id, name, datatype, path='', startDate='', endDate='',
755 754 startTime='', endTime='', parentId=None, server=None, **kwargs):
756 755
757 #Compatible with old signal chain version
758 if datatype==None and name==None:
756 # Compatible with old signal chain version
757 if datatype == None and name == None:
759 758 raise ValueError, 'datatype or name should be defined'
760
761 if name==None:
759
760 if name == None:
762 761 if 'Reader' in datatype:
763 762 name = datatype
764 763 else:
765 name = '%sReader' %(datatype)
766 if datatype==None:
767 datatype = name.replace('Reader','')
764 name = '%sReader' % (datatype)
765 if datatype == None:
766 datatype = name.replace('Reader', '')
768 767
769 768 self.id = id
770 769 self.name = name
771 770 self.datatype = datatype
772 771 if path != '':
773 772 self.path = os.path.abspath(path)
774 773 self.startDate = startDate
775 774 self.endDate = endDate
776 775 self.startTime = startTime
777 776 self.endTime = endTime
778 777 self.inputId = '0'
779 778 self.parentId = parentId
780 779 self.server = server
781 780 self.addRunOperation(**kwargs)
782 781
783 782 def update(self, **kwargs):
784 783
785 784 if 'datatype' in kwargs:
786 785 datatype = kwargs.pop('datatype')
787 786 if 'Reader' in datatype:
788 787 self.name = datatype
789 788 else:
790 self.name = '%sReader' %(datatype)
789 self.name = '%sReader' % (datatype)
791 790 self.datatype = self.name.replace('Reader', '')
792 791
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
794
792 attrs = ('path', 'startDate', 'endDate',
793 'startTime', 'endTime', 'parentId')
794
795 795 for attr in attrs:
796 796 if attr in kwargs:
797 797 setattr(self, attr, kwargs.pop(attr))
798
798
799 799 self.inputId = '0'
800 800 self.updateRunOperation(**kwargs)
801 801
802 802 def removeOperations(self):
803 803
804 804 for obj in self.opConfObjList:
805 805 del obj
806 806
807 807 self.opConfObjList = []
808 808
809 809 def addRunOperation(self, **kwargs):
810 810
811 opObj = self.addOperation(name = 'run', optype = 'self')
811 opObj = self.addOperation(name='run', optype='self')
812 812
813 813 if self.server is None:
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
814 opObj.addParameter(
815 name='datatype', value=self.datatype, format='str')
815 816 opObj.addParameter(name='path', value=self.path, format='str')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
820
817 opObj.addParameter(
818 name='startDate', value=self.startDate, format='date')
819 opObj.addParameter(
820 name='endDate', value=self.endDate, format='date')
821 opObj.addParameter(
822 name='startTime', value=self.startTime, format='time')
823 opObj.addParameter(
824 name='endTime', value=self.endTime, format='time')
825
821 826 for key, value in kwargs.items():
822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
827 opObj.addParameter(name=key, value=value,
828 format=type(value).__name__)
823 829 else:
824 opObj.addParameter(name='server' , value=self.server, format='str')
825
830 opObj.addParameter(name='server', value=self.server, format='str')
826 831
827 832 return opObj
828 833
829 834 def updateRunOperation(self, **kwargs):
830 835
831 836 opObj = self.getOperationObj(name='run')
832 837 opObj.removeParameters()
833 838
834 839 opObj.addParameter(name='datatype', value=self.datatype, format='str')
835 840 opObj.addParameter(name='path', value=self.path, format='str')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
841 opObj.addParameter(
842 name='startDate', value=self.startDate, format='date')
837 843 opObj.addParameter(name='endDate', value=self.endDate, format='date')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
844 opObj.addParameter(
845 name='startTime', value=self.startTime, format='time')
839 846 opObj.addParameter(name='endTime', value=self.endTime, format='time')
840
847
841 848 for key, value in kwargs.items():
842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
849 opObj.addParameter(name=key, value=value,
850 format=type(value).__name__)
843 851
844 852 return opObj
845
853
846 854 def readXml(self, upElement):
847 855
848 856 self.id = upElement.get('id')
849 857 self.name = upElement.get('name')
850 858 self.datatype = upElement.get('datatype')
851 859 self.inputId = upElement.get('inputId')
852 860
853 861 if self.ELEMENTNAME == 'ReadUnit':
854 862 self.datatype = self.datatype.replace('Reader', '')
855 863
856 864 if self.inputId == 'None':
857 865 self.inputId = '0'
858 866
859 867 self.opConfObjList = []
860 868
861 869 opElementList = upElement.iter(OperationConf().getElementName())
862 870
863 871 for opElement in opElementList:
864 872 opConfObj = OperationConf()
865 873 opConfObj.readXml(opElement)
866 874 self.opConfObjList.append(opConfObj)
867 875
868 876 if opConfObj.name == 'run':
869 877 self.path = opConfObj.getParameterValue('path')
870 878 self.startDate = opConfObj.getParameterValue('startDate')
871 879 self.endDate = opConfObj.getParameterValue('endDate')
872 880 self.startTime = opConfObj.getParameterValue('startTime')
873 881 self.endTime = opConfObj.getParameterValue('endTime')
874 882
883
875 884 class Project(Process):
876 885
877 886 id = None
878 887 # name = None
879 888 description = None
880 889 filename = None
881 890
882 891 procUnitConfObjDict = None
883 892
884 893 ELEMENTNAME = 'Project'
885 894
886 895 plotterQueue = None
887 896
888 897 def __init__(self, plotter_queue=None):
889 898
890 899 Process.__init__(self)
891 900 self.id = None
892 901 # self.name = None
893 902 self.description = None
894 903
895 904 self.plotterQueue = plotter_queue
896 905
897 906 self.procUnitConfObjDict = {}
898 907
899 908 def __getNewId(self):
900 909
901 910 idList = self.procUnitConfObjDict.keys()
902 911
903 id = int(self.id)*10
912 id = int(self.id) * 10
904 913
905 914 while True:
906 915 id += 1
907 916
908 917 if str(id) in idList:
909 918 continue
910 919
911 920 break
912 921
913 922 return str(id)
914 923
915 924 def getElementName(self):
916 925
917 926 return self.ELEMENTNAME
918 927
919 928 def getId(self):
920 929
921 930 return self.id
922 931
923 932 def updateId(self, new_id):
924 933
925 934 self.id = str(new_id)
926 935
927 936 keyList = self.procUnitConfObjDict.keys()
928 937 keyList.sort()
929 938
930 939 n = 1
931 940 newProcUnitConfObjDict = {}
932 941
933 942 for procKey in keyList:
934 943
935 944 procUnitConfObj = self.procUnitConfObjDict[procKey]
936 idProcUnit = str(int(self.id)*10 + n)
937 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
945 idProcUnit = str(int(self.id) * 10 + n)
946 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
938 947
939 948 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
940 949 n += 1
941 950
942 951 self.procUnitConfObjDict = newProcUnitConfObjDict
943 952
944 953 def setup(self, id, name='', description=''):
945 954
946 955 print
947 print '*'*60
956 print '*' * 60
948 957 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
958 print '*' * 60
950 959 print
951 960 self.id = str(id)
952 961 self.description = description
953 962
954 963 def update(self, name, description):
955 964
956 965 self.description = description
957 966
958 967 def clone(self):
959 968
960 969 p = Project()
961 970 p.procUnitConfObjDict = self.procUnitConfObjDict
962 971 return p
963 972
964 973 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965 974
966 975 if id is None:
967 976 idReadUnit = self.__getNewId()
968 977 else:
969 978 idReadUnit = str(id)
970 979
971 980 readUnitConfObj = ReadUnitConf()
972 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
981 readUnitConfObj.setup(idReadUnit, name, datatype,
982 parentId=self.id, **kwargs)
973 983
974 984 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
975 985
976 986 return readUnitConfObj
977 987
978 988 def addProcUnit(self, inputId='0', datatype=None, name=None):
979 989
980 990 idProcUnit = self.__getNewId()
981 991
982 992 procUnitConfObj = ProcUnitConf()
983 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
993 procUnitConfObj.setup(idProcUnit, name, datatype,
994 inputId, parentId=self.id)
984 995
985 996 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
986 997
987 998 return procUnitConfObj
988 999
989 1000 def removeProcUnit(self, id):
990 1001
991 1002 if id in self.procUnitConfObjDict.keys():
992 1003 self.procUnitConfObjDict.pop(id)
993 1004
994 1005 def getReadUnitId(self):
995 1006
996 1007 readUnitConfObj = self.getReadUnitObj()
997 1008
998 1009 return readUnitConfObj.id
999 1010
1000 1011 def getReadUnitObj(self):
1001 1012
1002 1013 for obj in self.procUnitConfObjDict.values():
1003 1014 if obj.getElementName() == 'ReadUnit':
1004 1015 return obj
1005 1016
1006 1017 return None
1007 1018
1008 1019 def getProcUnitObj(self, id=None, name=None):
1009 1020
1010 1021 if id != None:
1011 1022 return self.procUnitConfObjDict[id]
1012 1023
1013 1024 if name != None:
1014 1025 return self.getProcUnitObjByName(name)
1015 1026
1016 1027 return None
1017 1028
1018 1029 def getProcUnitObjByName(self, name):
1019 1030
1020 1031 for obj in self.procUnitConfObjDict.values():
1021 1032 if obj.name == name:
1022 1033 return obj
1023 1034
1024 1035 return None
1025 1036
1026 1037 def procUnitItems(self):
1027 1038
1028 1039 return self.procUnitConfObjDict.items()
1029 1040
1030 1041 def makeXml(self):
1031 1042
1032 1043 projectElement = Element('Project')
1033 1044 projectElement.set('id', str(self.id))
1034 1045 projectElement.set('name', self.name)
1035 1046 projectElement.set('description', self.description)
1036 1047
1037 1048 for procUnitConfObj in self.procUnitConfObjDict.values():
1038 1049 procUnitConfObj.makeXml(projectElement)
1039 1050
1040 1051 self.projectElement = projectElement
1041 1052
1042 1053 def writeXml(self, filename=None):
1043 1054
1044 1055 if filename == None:
1045 1056 if self.filename:
1046 1057 filename = self.filename
1047 1058 else:
1048 1059 filename = 'schain.xml'
1049 1060
1050 1061 if not filename:
1051 1062 print 'filename has not been defined. Use setFilename(filename) for do it.'
1052 1063 return 0
1053 1064
1054 1065 abs_file = os.path.abspath(filename)
1055 1066
1056 1067 if not os.access(os.path.dirname(abs_file), os.W_OK):
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1068 print 'No write permission on %s' % os.path.dirname(abs_file)
1058 1069 return 0
1059 1070
1060 1071 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1072 print 'File %s already exists and it could not be overwriten' % abs_file
1062 1073 return 0
1063 1074
1064 1075 self.makeXml()
1065 1076
1066 1077 ElementTree(self.projectElement).write(abs_file, method='xml')
1067 1078
1068 1079 self.filename = abs_file
1069 1080
1070 1081 return 1
1071 1082
1072 def readXml(self, filename = None):
1083 def readXml(self, filename=None):
1073 1084
1074 1085 if not filename:
1075 1086 print 'filename is not defined'
1076 1087 return 0
1077 1088
1078 1089 abs_file = os.path.abspath(filename)
1079 1090
1080 1091 if not os.path.isfile(abs_file):
1081 print '%s file does not exist' %abs_file
1092 print '%s file does not exist' % abs_file
1082 1093 return 0
1083 1094
1084 1095 self.projectElement = None
1085 1096 self.procUnitConfObjDict = {}
1086 1097
1087 1098 try:
1088 1099 self.projectElement = ElementTree().parse(abs_file)
1089 1100 except:
1090 print 'Error reading %s, verify file format' %filename
1101 print 'Error reading %s, verify file format' % filename
1091 1102 return 0
1092 1103
1093 1104 self.project = self.projectElement.tag
1094 1105
1095 1106 self.id = self.projectElement.get('id')
1096 1107 self.name = self.projectElement.get('name')
1097 self.description = self.projectElement.get('description')
1098
1099 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1108 self.description = self.projectElement.get('description')
1109
1110 readUnitElementList = self.projectElement.iter(
1111 ReadUnitConf().getElementName())
1100 1112
1101 1113 for readUnitElement in readUnitElementList:
1102 1114 readUnitConfObj = ReadUnitConf()
1103 1115 readUnitConfObj.readXml(readUnitElement)
1104 1116
1105 1117 if readUnitConfObj.parentId == None:
1106 1118 readUnitConfObj.parentId = self.id
1107 1119
1108 1120 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1109 1121
1110 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1122 procUnitElementList = self.projectElement.iter(
1123 ProcUnitConf().getElementName())
1111 1124
1112 1125 for procUnitElement in procUnitElementList:
1113 1126 procUnitConfObj = ProcUnitConf()
1114 1127 procUnitConfObj.readXml(procUnitElement)
1115 1128
1116 1129 if procUnitConfObj.parentId == None:
1117 1130 procUnitConfObj.parentId = self.id
1118 1131
1119 1132 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1120 1133
1121 1134 self.filename = abs_file
1122 1135
1123 1136 return 1
1124 1137
1125 1138 def printattr(self):
1126 1139
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1128 self.name,
1129 self.description)
1130
1140 print 'Project[%s]: name = %s, description = %s' % (self.id,
1141 self.name,
1142 self.description)
1143
1131 1144 for procUnitConfObj in self.procUnitConfObjDict.values():
1132 1145 procUnitConfObj.printattr()
1133 1146
1134 1147 def createObjects(self):
1135 1148
1136 1149 for procUnitConfObj in self.procUnitConfObjDict.values():
1137 1150 procUnitConfObj.createObjects(self.plotterQueue)
1138 1151
1139 1152 def __connect(self, objIN, thisObj):
1140 1153
1141 1154 thisObj.setInput(objIN.getOutputObj())
1142 1155
1143 1156 def connectObjects(self):
1144 1157
1145 1158 for thisPUConfObj in self.procUnitConfObjDict.values():
1146 1159
1147 1160 inputId = thisPUConfObj.getInputId()
1148 1161
1149 1162 if int(inputId) == 0:
1150 1163 continue
1151 1164
1152 #Get input object
1165 # Get input object
1153 1166 puConfINObj = self.procUnitConfObjDict[inputId]
1154 1167 puObjIN = puConfINObj.getProcUnitObj()
1155 1168
1156 #Get current object
1169 # Get current object
1157 1170 thisPUObj = thisPUConfObj.getProcUnitObj()
1158 1171
1159 1172 self.__connect(puObjIN, thisPUObj)
1160 1173
1161 1174 def __handleError(self, procUnitConfObj, send_email=False):
1162 1175
1163 1176 import socket
1164 1177
1165 1178 err = traceback.format_exception(sys.exc_info()[0],
1166 sys.exc_info()[1],
1167 sys.exc_info()[2])
1168
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1170 print '***** %s' %err[-1]
1179 sys.exc_info()[1],
1180 sys.exc_info()[2])
1181
1182 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1183 print '***** %s' % err[-1]
1171 1184
1172 1185 message = ''.join(err)
1173 1186
1174 1187 sys.stderr.write(message)
1175 1188
1176 1189 if not send_email:
1177 1190 return
1178 1191
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1192 subject = 'SChain v%s: Error running %s\n' % (
1193 schainpy.__version__, procUnitConfObj.name)
1180 1194
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1184 subtitle += 'Configuration file: %s\n' %self.filename
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1195 subtitle = '%s: %s\n' % (
1196 procUnitConfObj.getElementName(), procUnitConfObj.name)
1197 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1198 socket.gethostname())
1199 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1200 subtitle += 'Configuration file: %s\n' % self.filename
1201 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1186 1202
1187 1203 readUnitConfObj = self.getReadUnitObj()
1188 1204 if readUnitConfObj:
1189 1205 subtitle += '\nInput parameters:\n'
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1206 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1207 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1208 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1209 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1210 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1211 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1196 1212
1197 1213 adminObj = schainpy.admin.SchainNotify()
1198 1214 adminObj.sendAlert(message=message,
1199 subject=subject,
1200 subtitle=subtitle,
1201 filename=self.filename)
1202
1215 subject=subject,
1216 subtitle=subtitle,
1217 filename=self.filename)
1218
1203 1219 def isPaused(self):
1204 1220 return 0
1205 1221
1206 1222 def isStopped(self):
1207 1223 return 0
1208 1224
1209 1225 def runController(self):
1210 1226 '''
1211 1227 returns 0 when this process has been stopped, 1 otherwise
1212 1228 '''
1213 1229
1214 1230 if self.isPaused():
1215 1231 print 'Process suspended'
1216 1232
1217 1233 while True:
1218 1234 time.sleep(0.1)
1219 1235
1220 1236 if not self.isPaused():
1221 1237 break
1222 1238
1223 1239 if self.isStopped():
1224 1240 break
1225 1241
1226 1242 print 'Process reinitialized'
1227 1243
1228 1244 if self.isStopped():
1229 1245 print 'Process stopped'
1230 1246 return 0
1231 1247
1232 1248 return 1
1233 1249
1234 1250 def setFilename(self, filename):
1235 1251
1236 1252 self.filename = filename
1237 1253
1238 1254 def setPlotterQueue(self, plotter_queue):
1239 1255
1240 1256 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1241 1257
1242 1258 def getPlotterQueue(self):
1243 1259
1244 1260 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1245 1261
1246 1262 def useExternalPlotter(self):
1247 1263
1248 1264 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1249 1265
1250 1266 def run(self):
1251 1267
1252 1268 log.success('Starting {}'.format(self.name))
1253
1269
1254 1270 self.createObjects()
1255 1271 self.connectObjects()
1256 1272
1257 1273 keyList = self.procUnitConfObjDict.keys()
1258 1274 keyList.sort()
1259 1275
1260 1276 while(True):
1261 1277
1262 1278 is_ok = False
1263 1279
1264 1280 for procKey in keyList:
1265 1281
1266 1282 procUnitConfObj = self.procUnitConfObjDict[procKey]
1267 1283
1268 1284 try:
1269 1285 sts = procUnitConfObj.run()
1270 1286 is_ok = is_ok or sts
1271 1287 except KeyboardInterrupt:
1272 1288 is_ok = False
1273 1289 break
1274 1290 except ValueError, e:
1275 1291 time.sleep(0.5)
1276 1292 self.__handleError(procUnitConfObj, send_email=True)
1277 1293 is_ok = False
1278 1294 break
1279 1295 except:
1280 1296 time.sleep(0.5)
1281 1297 self.__handleError(procUnitConfObj)
1282 1298 is_ok = False
1283 1299 break
1284 1300
1285 #If every process unit finished so end process
1301 # If every process unit finished so end process
1286 1302 if not(is_ok):
1287 1303 break
1288 1304
1289 1305 if not self.runController():
1290 1306 break
1291 1307
1292 #Closing every process
1308 # Closing every process
1293 1309 for procKey in keyList:
1294 1310 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 1311 procUnitConfObj.close()
1296 1312
1297 1313 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
@@ -1,736 +1,762
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6
7 7 import numpy
8 8
9 9 from jroIO_base import LOCALTIME, JRODataReader, JRODataWriter
10 10 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
11 11 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
12 12 from schainpy.model.data.jrodata import Voltage
13 13 import zmq
14 14 import tempfile
15 15 from StringIO import StringIO
16 16 # from _sha import blocksize
17 17
18
18 19 class VoltageReader(JRODataReader, ProcessingUnit):
19 20 """
20 21 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
21 22 de los datos siempre se realiza por bloques. Los datos leidos (array de 3 dimensiones:
22 23 perfiles*alturas*canales) son almacenados en la variable "buffer".
23 24
24 25 perfiles * alturas * canales
25 26
26 27 Esta clase contiene instancias (objetos) de las clases BasicHeader, SystemHeader,
27 28 RadarControllerHeader y Voltage. Los tres primeros se usan para almacenar informacion de la
28 29 cabecera de datos (metadata), y el cuarto (Voltage) para obtener y almacenar un perfil de
29 30 datos desde el "buffer" cada vez que se ejecute el metodo "getData".
30 31
31 32 Example:
32 33
33 34 dpath = "/home/myuser/data"
34 35
35 36 startTime = datetime.datetime(2010,1,20,0,0,0,0,0,0)
36 37
37 38 endTime = datetime.datetime(2010,1,21,23,59,59,0,0,0)
38 39
39 40 readerObj = VoltageReader()
40 41
41 42 readerObj.setup(dpath, startTime, endTime)
42 43
43 44 while(True):
44 45
45 46 #to get one profile
46 47 profile = readerObj.getData()
47 48
48 49 #print the profile
49 50 print profile
50 51
51 52 #If you want to see all datablock
52 53 print readerObj.datablock
53 54
54 55 if readerObj.flagNoMoreFiles:
55 56 break
56 57
57 58 """
58 59
59 60 ext = ".r"
60 61
61 62 optchar = "D"
62 63 dataOut = None
63 64
64 65 def __init__(self, **kwargs):
65 66 """
66 67 Inicializador de la clase VoltageReader para la lectura de datos de voltage.
67 68
68 69 Input:
69 70 dataOut : Objeto de la clase Voltage. Este objeto sera utilizado para
70 71 almacenar un perfil de datos cada vez que se haga un requerimiento
71 72 (getData). El perfil sera obtenido a partir del buffer de datos,
72 73 si el buffer esta vacio se hara un nuevo proceso de lectura de un
73 74 bloque de datos.
74 75 Si este parametro no es pasado se creara uno internamente.
75 76
76 77 Variables afectadas:
77 78 self.dataOut
78 79
79 80 Return:
80 81 None
81 82 """
82 83
83 84 ProcessingUnit.__init__(self, **kwargs)
84 85
85 86 self.isConfig = False
86 87
87 88 self.datablock = None
88 89
89 90 self.utc = 0
90 91
91 92 self.ext = ".r"
92 93
93 94 self.optchar = "D"
94 95
95 96 self.basicHeaderObj = BasicHeader(LOCALTIME)
96 97
97 98 self.systemHeaderObj = SystemHeader()
98 99
99 100 self.radarControllerHeaderObj = RadarControllerHeader()
100 101
101 102 self.processingHeaderObj = ProcessingHeader()
102 103
103 104 self.online = 0
104 105
105 106 self.fp = None
106 107
107 108 self.idFile = None
108 109
109 110 self.dtype = None
110 111
111 112 self.fileSizeByHeader = None
112 113
113 114 self.filenameList = []
114 115
115 116 self.filename = None
116 117
117 118 self.fileSize = None
118 119
119 120 self.firstHeaderSize = 0
120 121
121 122 self.basicHeaderSize = 24
122 123
123 124 self.pathList = []
124 125
125 126 self.filenameList = []
126 127
127 128 self.lastUTTime = 0
128 129
129 130 self.maxTimeStep = 30
130 131
131 132 self.flagNoMoreFiles = 0
132 133
133 134 self.set = 0
134 135
135 136 self.path = None
136 137
137 self.profileIndex = 2**32-1
138 self.profileIndex = 2**32 - 1
138 139
139 self.delay = 3 #seconds
140 self.delay = 3 # seconds
140 141
141 self.nTries = 3 #quantity tries
142 self.nTries = 3 # quantity tries
142 143
143 self.nFiles = 3 #number of files for searching
144 self.nFiles = 3 # number of files for searching
144 145
145 146 self.nReadBlocks = 0
146 147
147 148 self.flagIsNewFile = 1
148 149
149 150 self.__isFirstTimeOnline = 1
150 151
151 152 # self.ippSeconds = 0
152 153
153 154 self.flagDiscontinuousBlock = 0
154 155
155 156 self.flagIsNewBlock = 0
156 157
157 158 self.nTotalBlocks = 0
158 159
159 160 self.blocksize = 0
160 161
161 162 self.dataOut = self.createObjByDefault()
162 163
163 164 self.nTxs = 1
164 165
165 166 self.txIndex = 0
166 167
167 168 def createObjByDefault(self):
168 169
169 170 dataObj = Voltage()
170 171
171 172 return dataObj
172 173
173 174 def __hasNotDataInBuffer(self):
174 175
175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
176 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
176 177 return 1
177 178
178 179 return 0
179 180
180
181 181 def getBlockDimension(self):
182 182 """
183 183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184
184
185 185 Affected:
186 186 self.blocksize
187 187
188 188 Return:
189 189 None
190 190 """
191 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
191 pts2read = self.processingHeaderObj.profilesPerBlock * \
192 self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
192 193 self.blocksize = pts2read
193 194
194
195
196 195 def readBlock(self):
197 196 """
198 197 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
199 198 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
200 199 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
201 200 es seteado a 0
202
201
203 202 Inputs:
204 203 None
205
204
206 205 Return:
207 206 None
208
207
209 208 Affected:
210 209 self.profileIndex
211 210 self.datablock
212 211 self.flagIsNewFile
213 212 self.flagIsNewBlock
214 213 self.nTotalBlocks
215
214
216 215 Exceptions:
217 216 Si un bloque leido no es un bloque valido
218 217 """
219
218
220 219 # if self.server is not None:
221 220 # self.zBlock = self.receiver.recv()
222 221 # self.zHeader = self.zBlock[:24]
223 222 # self.zDataBlock = self.zBlock[24:]
224 223 # junk = numpy.fromstring(self.zDataBlock, numpy.dtype([('real','<i4'),('imag','<i4')]))
225 224 # self.processingHeaderObj.profilesPerBlock = 240
226 225 # self.processingHeaderObj.nHeights = 248
227 226 # self.systemHeaderObj.nChannels
228 227 # else:
229 228 current_pointer_location = self.fp.tell()
230 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
229 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
231 230
232 231 try:
233 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
232 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
233 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
234 234 except:
235 #print "The read block (%3d) has not enough data" %self.nReadBlocks
235 # print "The read block (%3d) has not enough data" %self.nReadBlocks
236 236
237 237 if self.waitDataBlock(pointer_location=current_pointer_location):
238 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
239 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
238 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
239 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
240 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
240 241 # return 0
241 242
242 #Dimensions : nChannels, nProfiles, nSamples
243 # Dimensions : nChannels, nProfiles, nSamples
243 244
244 junk = numpy.transpose(junk, (2,0,1))
245 self.datablock = junk['real'] + junk['imag']*1j
245 junk = numpy.transpose(junk, (2, 0, 1))
246 self.datablock = junk['real'] + junk['imag'] * 1j
246 247
247 248 self.profileIndex = 0
248 249
249 250 self.flagIsNewFile = 0
250 251 self.flagIsNewBlock = 1
251 252
252 253 self.nTotalBlocks += 1
253 254 self.nReadBlocks += 1
254 255
255 256 return 1
256 257
257 258 def getFirstHeader(self):
258 259
259 260 self.getBasicHeader()
260 261
261 262 self.dataOut.systemHeaderObj = self.systemHeaderObj.copy()
262 263
263 264 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
264 265
265 266 if self.nTxs > 1:
266 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
267 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
267 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
268 # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
268 269
269 270 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
270 271 #
271 272 # if self.radarControllerHeaderObj.code is not None:
272 273 #
273 274 # self.dataOut.nCode = self.radarControllerHeaderObj.nCode
274 275 #
275 276 # self.dataOut.nBaud = self.radarControllerHeaderObj.nBaud
276 277 #
277 278 # self.dataOut.code = self.radarControllerHeaderObj.code
278 279
279 280 self.dataOut.dtype = self.dtype
280 281
281 282 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
282 283
283 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
284 self.dataOut.heightList = numpy.arange(
285 self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
284 286
285 287 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
286 288
287 289 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
288 290
289 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
291 # asumo q la data no esta decodificada
292 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
290 293
291 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
294 # asumo q la data no esta sin flip
295 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
292 296
293 297 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
294 298
295 299 def reshapeData(self):
296 300
297 301 if self.nTxs < 0:
298 302 return
299 303
300 304 if self.nTxs == 1:
301 305 return
302 306
303 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
304 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
307 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
308 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
309 1. / self.nTxs, self.processingHeaderObj.profilesPerBlock)
305 310
306 311 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
307 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
312 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" % (
313 self.nTxs, self.processingHeaderObj.nHeights)
308 314
309 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
315 self.datablock = self.datablock.reshape(
316 (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, self.processingHeaderObj.nHeights / self.nTxs))
310 317
311 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
312 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
313 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
318 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
319 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
320 self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
321 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
314 322
315 323 return
316 324
317 325 def readFirstHeaderFromServer(self):
318
326
319 327 self.getFirstHeader()
320 328
321 329 self.firstHeaderSize = self.basicHeaderObj.size
322 330
323 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
331 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
332 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
324 333 if datatype == 0:
325 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
334 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
326 335 elif datatype == 1:
327 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
336 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
328 337 elif datatype == 2:
329 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
338 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
330 339 elif datatype == 3:
331 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
340 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
332 341 elif datatype == 4:
333 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
342 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
334 343 elif datatype == 5:
335 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
344 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
336 345 else:
337 346 raise ValueError, 'Data type was not defined'
338 347
339 348 self.dtype = datatype_str
340 349 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
341 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
350 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
351 self.firstHeaderSize + self.basicHeaderSize * \
352 (self.processingHeaderObj.dataBlocksPerFile - 1)
342 353 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
343 354 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
344 355 self.getBlockDimension()
345 356
346
347 def getFromServer(self):
357 def getFromServer(self):
348 358 self.flagDiscontinuousBlock = 0
349 359 self.profileIndex = 0
350 360 self.flagIsNewBlock = 1
351 361 self.dataOut.flagNoData = False
352 362 self.nTotalBlocks += 1
353 363 self.nReadBlocks += 1
354 364 self.blockPointer = 0
355 365
356 366 block = self.receiver.recv()
357 367
358 368 self.basicHeaderObj.read(block[self.blockPointer:])
359 369 self.blockPointer += self.basicHeaderObj.length
360 370 self.systemHeaderObj.read(block[self.blockPointer:])
361 371 self.blockPointer += self.systemHeaderObj.length
362 372 self.radarControllerHeaderObj.read(block[self.blockPointer:])
363 373 self.blockPointer += self.radarControllerHeaderObj.length
364 374 self.processingHeaderObj.read(block[self.blockPointer:])
365 375 self.blockPointer += self.processingHeaderObj.length
366 376 self.readFirstHeaderFromServer()
367
377
368 378 timestamp = self.basicHeaderObj.get_datatime()
369 379 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
370 380 current_pointer_location = self.blockPointer
371 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
381 junk = numpy.fromstring(
382 block[self.blockPointer:], self.dtype, self.blocksize)
372 383
373 384 try:
374 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
385 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
386 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
375 387 except:
376 #print "The read block (%3d) has not enough data" %self.nReadBlocks
388 # print "The read block (%3d) has not enough data" %self.nReadBlocks
377 389 if self.waitDataBlock(pointer_location=current_pointer_location):
378 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
379 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
390 junk = numpy.fromstring(
391 block[self.blockPointer:], self.dtype, self.blocksize)
392 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
393 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
380 394 # return 0
381 395
382 #Dimensions : nChannels, nProfiles, nSamples
396 # Dimensions : nChannels, nProfiles, nSamples
383 397
384 junk = numpy.transpose(junk, (2,0,1))
385 self.datablock = junk['real'] + junk['imag'] * 1j
398 junk = numpy.transpose(junk, (2, 0, 1))
399 self.datablock = junk['real'] + junk['imag'] * 1j
386 400 self.profileIndex = 0
387 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
401 if self.selBlocksize == None:
402 self.selBlocksize = self.dataOut.nProfiles
388 403 if self.selBlocktime != None:
389 404 if self.dataOut.nCohInt is not None:
390 405 nCohInt = self.dataOut.nCohInt
391 406 else:
392 407 nCohInt = 1
393 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
394 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
408 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
409 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
410 self.dataOut.data = self.datablock[:,
411 self.profileIndex:self.profileIndex + self.selBlocksize, :]
395 412 datasize = self.dataOut.data.shape[1]
396 413 if datasize < self.selBlocksize:
397 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
398 buffer[:,:datasize,:] = self.dataOut.data
414 buffer = numpy.zeros(
415 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
416 buffer[:, :datasize, :] = self.dataOut.data
399 417 self.dataOut.data = buffer
400 418 self.profileIndex = blockIndex
401 419
402 420 self.dataOut.flagDataAsBlock = True
403 421 self.flagIsNewBlock = 1
404 422 self.dataOut.realtime = self.online
405 423
406 424 return self.dataOut.data
407 425
408 426 def getData(self):
409 427 """
410 428 getData obtiene una unidad de datos del buffer de lectura, un perfil, y la copia al objeto self.dataOut
411 429 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
412 430 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
413 431 "readNextBlock"
414
432
415 433 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
416
434
417 435 Return:
418
436
419 437 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
420 438 es igual al total de perfiles leidos desde el archivo.
421
439
422 440 Si self.getByBlock == False:
423
441
424 442 self.dataOut.data = buffer[:, thisProfile, :]
425
443
426 444 shape = [nChannels, nHeis]
427
445
428 446 Si self.getByBlock == True:
429
447
430 448 self.dataOut.data = buffer[:, :, :]
431
449
432 450 shape = [nChannels, nProfiles, nHeis]
433
451
434 452 Variables afectadas:
435 453 self.dataOut
436 454 self.profileIndex
437
455
438 456 Affected:
439 457 self.dataOut
440 458 self.profileIndex
441 459 self.flagDiscontinuousBlock
442 460 self.flagIsNewBlock
443 461 """
444 462 if self.flagNoMoreFiles:
445 463 self.dataOut.flagNoData = True
446 464 print 'Process finished'
447 465 return 0
448 466 self.flagDiscontinuousBlock = 0
449 467 self.flagIsNewBlock = 0
450 468 if self.__hasNotDataInBuffer():
451 if not( self.readNextBlock() ):
469 if not(self.readNextBlock()):
452 470 return 0
453 471
454 472 self.getFirstHeader()
455 473
456 474 self.reshapeData()
457 475 if self.datablock is None:
458 476 self.dataOut.flagNoData = True
459 477 return 0
460 478
461 479 if not self.getByBlock:
462 480
463 481 """
464 482 Return profile by profile
465 483
466 484 If nTxs > 1 then one profile is divided by nTxs and number of total
467 485 blocks is increased by nTxs (nProfiles *= nTxs)
468 486 """
469 487 self.dataOut.flagDataAsBlock = False
470 self.dataOut.data = self.datablock[:,self.profileIndex,:]
488 self.dataOut.data = self.datablock[:, self.profileIndex, :]
471 489 self.dataOut.profileIndex = self.profileIndex
472 490
473 491 self.profileIndex += 1
474
492
475 493 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
476 494 # """
477 495 # Return all block
478 496 # """
479 497 # self.dataOut.flagDataAsBlock = True
480 498 # self.dataOut.data = self.datablock
481 499 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
482 #
500 #
483 501 # self.profileIndex = self.dataOut.nProfiles
484
502
485 503 else:
486 504 """
487 505 Return a block
488 506 """
489 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
507 if self.selBlocksize == None:
508 self.selBlocksize = self.dataOut.nProfiles
490 509 if self.selBlocktime != None:
491 510 if self.dataOut.nCohInt is not None:
492 511 nCohInt = self.dataOut.nCohInt
493 512 else:
494 513 nCohInt = 1
495 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
514 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
515 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
496 516
497 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
517 self.dataOut.data = self.datablock[:,
518 self.profileIndex:self.profileIndex + self.selBlocksize, :]
498 519 self.profileIndex += self.selBlocksize
499 520 datasize = self.dataOut.data.shape[1]
500 521
501 522 if datasize < self.selBlocksize:
502 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
503 buffer[:,:datasize,:] = self.dataOut.data
523 buffer = numpy.zeros(
524 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
525 buffer[:, :datasize, :] = self.dataOut.data
504 526
505 while datasize < self.selBlocksize: #Not enough profiles to fill the block
506 if not( self.readNextBlock() ):
527 while datasize < self.selBlocksize: # Not enough profiles to fill the block
528 if not(self.readNextBlock()):
507 529 return 0
508 530 self.getFirstHeader()
509 531 self.reshapeData()
510 532 if self.datablock is None:
511 533 self.dataOut.flagNoData = True
512 534 return 0
513 #stack data
535 # stack data
514 536 blockIndex = self.selBlocksize - datasize
515 datablock1 = self.datablock[:,:blockIndex,:]
537 datablock1 = self.datablock[:, :blockIndex, :]
516 538
517 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
539 buffer[:, datasize:datasize +
540 datablock1.shape[1], :] = datablock1
518 541 datasize += datablock1.shape[1]
519 542
520 543 self.dataOut.data = buffer
521 544 self.profileIndex = blockIndex
522 545
523 546 self.dataOut.flagDataAsBlock = True
524 547 self.dataOut.nProfiles = self.dataOut.data.shape[1]
525 548
526 549 self.dataOut.flagNoData = False
527 550
528 551 self.getBasicHeader()
529 552
530 553 self.dataOut.realtime = self.online
531 554
532 555 return self.dataOut.data
533 556
557
534 558 class VoltageWriter(JRODataWriter, Operation):
535 559 """
536 560 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
537 561 de los datos siempre se realiza por bloques.
538 562 """
539 563
540 564 ext = ".r"
541 565
542 566 optchar = "D"
543 567
544 568 shapeBuffer = None
545 569
546
547 570 def __init__(self, **kwargs):
548 571 """
549 572 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
550 573
551 574 Affected:
552 575 self.dataOut
553 576
554 577 Return: None
555 578 """
556 579 Operation.__init__(self, **kwargs)
557 580
558 581 self.nTotalBlocks = 0
559 582
560 583 self.profileIndex = 0
561 584
562 585 self.isConfig = False
563 586
564 587 self.fp = None
565 588
566 589 self.flagIsNewFile = 1
567 590
568 591 self.blockIndex = 0
569 592
570 593 self.flagIsNewBlock = 0
571 594
572 595 self.setFile = None
573 596
574 597 self.dtype = None
575 598
576 599 self.path = None
577 600
578 601 self.filename = None
579 602
580 603 self.basicHeaderObj = BasicHeader(LOCALTIME)
581 604
582 605 self.systemHeaderObj = SystemHeader()
583 606
584 607 self.radarControllerHeaderObj = RadarControllerHeader()
585 608
586 609 self.processingHeaderObj = ProcessingHeader()
587 610
588 611 def hasAllDataInBuffer(self):
589 612 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock:
590 613 return 1
591 614 return 0
592 615
593
594 616 def setBlockDimension(self):
595 617 """
596 618 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
597 619
598 620 Affected:
599 621 self.shape_spc_Buffer
600 622 self.shape_cspc_Buffer
601 623 self.shape_dc_Buffer
602 624
603 625 Return: None
604 626 """
605 627 self.shapeBuffer = (self.processingHeaderObj.profilesPerBlock,
606 628 self.processingHeaderObj.nHeights,
607 629 self.systemHeaderObj.nChannels)
608 630
609 631 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
610 self.processingHeaderObj.profilesPerBlock,
611 self.processingHeaderObj.nHeights),
632 self.processingHeaderObj.profilesPerBlock,
633 self.processingHeaderObj.nHeights),
612 634 dtype=numpy.dtype('complex64'))
613 635
614 636 def writeBlock(self):
615 637 """
616 638 Escribe el buffer en el file designado
617 639
618 640 Affected:
619 641 self.profileIndex
620 642 self.flagIsNewFile
621 643 self.flagIsNewBlock
622 644 self.nTotalBlocks
623 645 self.blockIndex
624 646
625 647 Return: None
626 648 """
627 data = numpy.zeros( self.shapeBuffer, self.dtype )
649 data = numpy.zeros(self.shapeBuffer, self.dtype)
628 650
629 junk = numpy.transpose(self.datablock, (1,2,0))
651 junk = numpy.transpose(self.datablock, (1, 2, 0))
630 652
631 653 data['real'] = junk.real
632 654 data['imag'] = junk.imag
633 655
634 data = data.reshape( (-1) )
656 data = data.reshape((-1))
635 657
636 data.tofile( self.fp )
658 data.tofile(self.fp)
637 659
638 660 self.datablock.fill(0)
639 661
640 662 self.profileIndex = 0
641 663 self.flagIsNewFile = 0
642 664 self.flagIsNewBlock = 1
643 665
644 666 self.blockIndex += 1
645 667 self.nTotalBlocks += 1
646 668
647 669 # print "[Writing] Block = %04d" %self.blockIndex
648 670
649 671 def putData(self):
650 672 """
651 673 Setea un bloque de datos y luego los escribe en un file
652 674
653 675 Affected:
654 676 self.flagIsNewBlock
655 677 self.profileIndex
656 678
657 679 Return:
658 680 0 : Si no hay data o no hay mas files que puedan escribirse
659 681 1 : Si se escribio la data de un bloque en un file
660 682 """
661 683 if self.dataOut.flagNoData:
662 684 return 0
663 685
664 686 self.flagIsNewBlock = 0
665 687
666 688 if self.dataOut.flagDiscontinuousBlock:
667 689 self.datablock.fill(0)
668 690 self.profileIndex = 0
669 691 self.setNextFile()
670 692
671 693 if self.profileIndex == 0:
672 694 self.setBasicHeader()
673 695
674 self.datablock[:,self.profileIndex,:] = self.dataOut.data
696 self.datablock[:, self.profileIndex, :] = self.dataOut.data
675 697
676 698 self.profileIndex += 1
677 699
678 700 if self.hasAllDataInBuffer():
679 #if self.flagIsNewFile:
701 # if self.flagIsNewFile:
680 702 self.writeNextBlock()
681 703 # self.setFirstHeader()
682 704
683 705 return 1
684 706
685 707 def __getBlockSize(self):
686 708 '''
687 709 Este metodos determina el cantidad de bytes para un bloque de datos de tipo Voltage
688 710 '''
689 711
690 712 dtype_width = self.getDtypeWidth()
691 713
692 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
714 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
715 self.profilesPerBlock * dtype_width * 2)
693 716
694 717 return blocksize
695 718
696 719 def setFirstHeader(self):
697
698 720 """
699 721 Obtiene una copia del First Header
700 722
701 723 Affected:
702 724 self.systemHeaderObj
703 725 self.radarControllerHeaderObj
704 726 self.dtype
705 727
706 728 Return:
707 729 None
708 730 """
709 731
710 732 self.systemHeaderObj = self.dataOut.systemHeaderObj.copy()
711 733 self.systemHeaderObj.nChannels = self.dataOut.nChannels
712 734 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
713 735
714 self.processingHeaderObj.dtype = 0 # Voltage
736 self.processingHeaderObj.dtype = 0 # Voltage
715 737 self.processingHeaderObj.blockSize = self.__getBlockSize()
716 738 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
717 739 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
718 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
740 # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
741 self.processingHeaderObj.nWindows = 1
719 742 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
720 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
721 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
743 # Cuando la data de origen es de tipo Voltage
744 self.processingHeaderObj.nIncohInt = 1
745 # Cuando la data de origen es de tipo Voltage
746 self.processingHeaderObj.totalSpectra = 0
722 747
723 748 if self.dataOut.code is not None:
724 749 self.processingHeaderObj.code = self.dataOut.code
725 750 self.processingHeaderObj.nCode = self.dataOut.nCode
726 751 self.processingHeaderObj.nBaud = self.dataOut.nBaud
727 752
728 753 if self.processingHeaderObj.nWindows != 0:
729 754 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
730 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
755 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
756 self.dataOut.heightList[0]
731 757 self.processingHeaderObj.nHeights = self.dataOut.nHeights
732 758 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
733 759
734 760 self.processingHeaderObj.processFlags = self.getProcessFlags()
735 761
736 762 self.setBasicHeader()
General Comments 0
You need to be logged in to leave comments. Login now