##// END OF EJS Templates
schain
RSS
Bug fixed in Signal Chain GUI: Updating FTP configuration from old SendByFTP operation.
Miguel Valdez -
r607:bc5d3588d2df
parent child
Show More

The requested changes are too big and content was truncated. Show full diff

Show file before | Show file after | Hide comments
@@ -1,1209 +1,1209
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5 from xml.etree.ElementTree import Element, SubElement
6 6 from xml.etree import ElementTree as ET
7 7 from xml.dom import minidom
8 8
9 9 #import datetime
10 10 from model import *
11 11
12 12 try:
13 13 from gevent import sleep
14 14 except:
15 15 from time import sleep
16 16
17 17 import ast
18 18
19 19 def prettify(elem):
20 20 """Return a pretty-printed XML string for the Element.
21 21 """
22 22 rough_string = ET.tostring(elem, 'utf-8')
23 23 reparsed = minidom.parseString(rough_string)
24 24 return reparsed.toprettyxml(indent=" ")
25 25
26 26 class ParameterConf():
27 27
28 28 id = None
29 29 name = None
30 30 value = None
31 31 format = None
32 32
33 33 __formated_value = None
34 34
35 35 ELEMENTNAME = 'Parameter'
36 36
37 37 def __init__(self):
38 38
39 39 self.format = 'str'
40 40
41 41 def getElementName(self):
42 42
43 43 return self.ELEMENTNAME
44 44
45 45 def getValue(self):
46 46
47 47 value = self.value
48 48 format = self.format
49 49
50 50 if self.__formated_value != None:
51 51
52 52 return self.__formated_value
53 53
54 54 if format == 'str':
55 55 self.__formated_value = str(value)
56 56 return self.__formated_value
57 57
58 58 if value == '':
59 59 raise ValueError, "%s: This parameter value is empty" %self.name
60 60
61 61 if format == 'bool':
62 62 value = int(value)
63 63
64 64 if format == 'list':
65 65 strList = value.split(',')
66 66
67 67 self.__formated_value = strList
68 68
69 69 return self.__formated_value
70 70
71 71 if format == 'intlist':
72 72 """
73 73 Example:
74 74 value = (0,1,2)
75 75 """
76 76 value = value.replace('(', '')
77 77 value = value.replace(')', '')
78 78
79 79 value = value.replace('[', '')
80 80 value = value.replace(']', '')
81 81
82 82 strList = value.split(',')
83 83 intList = [int(x) for x in strList]
84 84
85 85 self.__formated_value = intList
86 86
87 87 return self.__formated_value
88 88
89 89 if format == 'floatlist':
90 90 """
91 91 Example:
92 92 value = (0.5, 1.4, 2.7)
93 93 """
94 94
95 95 value = value.replace('(', '')
96 96 value = value.replace(')', '')
97 97
98 98 value = value.replace('[', '')
99 99 value = value.replace(']', '')
100 100
101 101 strList = value.split(',')
102 102 floatList = [float(x) for x in strList]
103 103
104 104 self.__formated_value = floatList
105 105
106 106 return self.__formated_value
107 107
108 108 if format == 'date':
109 109 strList = value.split('/')
110 110 intList = [int(x) for x in strList]
111 111 date = datetime.date(intList[0], intList[1], intList[2])
112 112
113 113 self.__formated_value = date
114 114
115 115 return self.__formated_value
116 116
117 117 if format == 'time':
118 118 strList = value.split(':')
119 119 intList = [int(x) for x in strList]
120 120 time = datetime.time(intList[0], intList[1], intList[2])
121 121
122 122 self.__formated_value = time
123 123
124 124 return self.__formated_value
125 125
126 126 if format == 'pairslist':
127 127 """
128 128 Example:
129 129 value = (0,1),(1,2)
130 130 """
131 131
132 132 value = value.replace('(', '')
133 133 value = value.replace(')', '')
134 134
135 135 value = value.replace('[', '')
136 136 value = value.replace(']', '')
137 137
138 138 strList = value.split(',')
139 139 intList = [int(item) for item in strList]
140 140 pairList = []
141 141 for i in range(len(intList)/2):
142 142 pairList.append((intList[i*2], intList[i*2 + 1]))
143 143
144 144 self.__formated_value = pairList
145 145
146 146 return self.__formated_value
147 147
148 148 if format == 'multilist':
149 149 """
150 150 Example:
151 151 value = (0,1,2),(3,4,5)
152 152 """
153 153 multiList = ast.literal_eval(value)
154 154
155 155 if type(multiList[0]) == int:
156 156 multiList = ast.literal_eval("(" + value + ")")
157 157
158 158 self.__formated_value = multiList
159 159
160 160 return self.__formated_value
161 161
162 162 format_func = eval(format)
163 163
164 164 self.__formated_value = format_func(value)
165 165
166 166 return self.__formated_value
167 167
168 168 def updateId(self, new_id):
169 169
170 170 self.id = str(new_id)
171 171
172 172 def setup(self, id, name, value, format='str'):
173 173
174 174 self.id = str(id)
175 175 self.name = name
176 176 self.value = str(value)
177 177 self.format = str.lower(format)
178 178
179 179 def update(self, name, value, format='str'):
180 180
181 181 self.name = name
182 182 self.value = str(value)
183 183 self.format = format
184 184
185 185 def makeXml(self, opElement):
186 186
187 187 parmElement = SubElement(opElement, self.ELEMENTNAME)
188 188 parmElement.set('id', str(self.id))
189 189 parmElement.set('name', self.name)
190 190 parmElement.set('value', self.value)
191 191 parmElement.set('format', self.format)
192 192
193 193 def readXml(self, parmElement):
194 194
195 195 self.id = parmElement.get('id')
196 196 self.name = parmElement.get('name')
197 197 self.value = parmElement.get('value')
198 198 self.format = str.lower(parmElement.get('format'))
199 199
200 200 #Compatible with old signal chain version
201 201 if self.format == 'int' and self.name == 'idfigure':
202 202 self.name = 'id'
203 203
204 204 def printattr(self):
205 205
206 206 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
207 207
208 208 class OperationConf():
209 209
210 210 id = None
211 211 name = None
212 212 priority = None
213 213 type = None
214 214
215 215 parmConfObjList = []
216 216
217 217 ELEMENTNAME = 'Operation'
218 218
219 219 def __init__(self):
220 220
221 221 self.id = '0'
222 222 self.name = None
223 223 self.priority = None
224 224 self.type = 'self'
225 225
226 226
227 227 def __getNewId(self):
228 228
229 229 return int(self.id)*10 + len(self.parmConfObjList) + 1
230 230
231 231 def updateId(self, new_id):
232 232
233 233 self.id = str(new_id)
234 234
235 235 n = 1
236 236 for parmObj in self.parmConfObjList:
237 237
238 238 idParm = str(int(new_id)*10 + n)
239 239 parmObj.updateId(idParm)
240 240
241 241 n += 1
242 242
243 243 def getElementName(self):
244 244
245 245 return self.ELEMENTNAME
246 246
247 247 def getParameterObjList(self):
248 248
249 249 return self.parmConfObjList
250 250
251 251 def getParameterObj(self, parameterName):
252 252
253 253 for parmConfObj in self.parmConfObjList:
254 254
255 255 if parmConfObj.name != parameterName:
256 256 continue
257 257
258 258 return parmConfObj
259 259
260 260 return None
261 261
262 262 def getParameterObjfromValue(self, parameterValue):
263 263
264 264 for parmConfObj in self.parmConfObjList:
265 265
266 266 if parmConfObj.getValue() != parameterValue:
267 267 continue
268 268
269 269 return parmConfObj.getValue()
270 270
271 271 return None
272 272
273 273 def getParameterValue(self, parameterName):
274 274
275 275 parameterObj = self.getParameterObj(parameterName)
276 276
277 if not parameterObj:
278 return None
277 # if not parameterObj:
278 # return None
279 279
280 280 value = parameterObj.getValue()
281 281
282 282 return value
283 283
284 284 def setup(self, id, name, priority, type):
285 285
286 286 self.id = str(id)
287 287 self.name = name
288 288 self.type = type
289 289 self.priority = priority
290 290
291 291 self.parmConfObjList = []
292 292
293 293 def removeParameters(self):
294 294
295 295 for obj in self.parmConfObjList:
296 296 del obj
297 297
298 298 self.parmConfObjList = []
299 299
300 300 def addParameter(self, name, value, format='str'):
301 301
302 302 id = self.__getNewId()
303 303
304 304 parmConfObj = ParameterConf()
305 305 parmConfObj.setup(id, name, value, format)
306 306
307 307 self.parmConfObjList.append(parmConfObj)
308 308
309 309 return parmConfObj
310 310
311 311 def changeParameter(self, name, value, format='str'):
312 312
313 313 parmConfObj = self.getParameterObj(name)
314 314 parmConfObj.update(name, value, format)
315 315
316 316 return parmConfObj
317 317
318 318 def makeXml(self, upElement):
319 319
320 320 opElement = SubElement(upElement, self.ELEMENTNAME)
321 321 opElement.set('id', str(self.id))
322 322 opElement.set('name', self.name)
323 323 opElement.set('type', self.type)
324 324 opElement.set('priority', str(self.priority))
325 325
326 326 for parmConfObj in self.parmConfObjList:
327 327 parmConfObj.makeXml(opElement)
328 328
329 329 def readXml(self, opElement):
330 330
331 331 self.id = opElement.get('id')
332 332 self.name = opElement.get('name')
333 333 self.type = opElement.get('type')
334 334 self.priority = opElement.get('priority')
335 335
336 336 #Compatible with old signal chain version
337 337 #Use of 'run' method instead 'init'
338 338 if self.type == 'self' and self.name == 'init':
339 339 self.name = 'run'
340 340
341 341 self.parmConfObjList = []
342 342
343 343 parmElementList = opElement.getiterator(ParameterConf().getElementName())
344 344
345 345 for parmElement in parmElementList:
346 346 parmConfObj = ParameterConf()
347 347 parmConfObj.readXml(parmElement)
348 348
349 349 #Compatible with old signal chain version
350 350 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
351 351 if self.type != 'self' and self.name == 'Plot':
352 352 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
353 353 self.name = parmConfObj.value
354 354 continue
355 355
356 356 self.parmConfObjList.append(parmConfObj)
357 357
358 358 def printattr(self):
359 359
360 360 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
361 361 self.id,
362 362 self.name,
363 363 self.type,
364 364 self.priority)
365 365
366 366 for parmConfObj in self.parmConfObjList:
367 367 parmConfObj.printattr()
368 368
369 369 def createObject(self):
370 370
371 371 if self.type == 'self':
372 372 raise ValueError, "This operation type cannot be created"
373 373
374 374 if self.type == 'external' or self.type == 'other':
375 375 className = eval(self.name)
376 376 opObj = className()
377 377
378 378 return opObj
379 379
380 380 class ProcUnitConf():
381 381
382 382 id = None
383 383 name = None
384 384 datatype = None
385 385 inputId = None
386 386 parentId = None
387 387
388 388 opConfObjList = []
389 389
390 390 procUnitObj = None
391 391 opObjList = []
392 392
393 393 ELEMENTNAME = 'ProcUnit'
394 394
395 395 def __init__(self):
396 396
397 397 self.id = None
398 398 self.datatype = None
399 399 self.name = None
400 400 self.inputId = None
401 401
402 402 self.opConfObjList = []
403 403
404 404 self.procUnitObj = None
405 405 self.opObjDict = {}
406 406
407 407 def __getPriority(self):
408 408
409 409 return len(self.opConfObjList)+1
410 410
411 411 def __getNewId(self):
412 412
413 413 return int(self.id)*10 + len(self.opConfObjList) + 1
414 414
415 415 def getElementName(self):
416 416
417 417 return self.ELEMENTNAME
418 418
419 419 def getId(self):
420 420
421 421 return self.id
422 422
423 423 def updateId(self, new_id, parentId=parentId):
424 424
425 425
426 426 new_id = int(parentId)*10 + (int(self.id) % 10)
427 427 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
428 428
429 429 #If this proc unit has not inputs
430 430 if self.inputId == '0':
431 431 new_inputId = 0
432 432
433 433 n = 1
434 434 for opConfObj in self.opConfObjList:
435 435
436 436 idOp = str(int(new_id)*10 + n)
437 437 opConfObj.updateId(idOp)
438 438
439 439 n += 1
440 440
441 441 self.parentId = str(parentId)
442 442 self.id = str(new_id)
443 443 self.inputId = str(new_inputId)
444 444
445 445
446 446 def getInputId(self):
447 447
448 448 return self.inputId
449 449
450 450 def getOperationObjList(self):
451 451
452 452 return self.opConfObjList
453 453
454 454 def getOperationObj(self, name=None):
455 455
456 456 for opConfObj in self.opConfObjList:
457 457
458 458 if opConfObj.name != name:
459 459 continue
460 460
461 461 return opConfObj
462 462
463 463 return None
464 464
465 465 def getOpObjfromParamValue(self, value=None):
466 466
467 467 for opConfObj in self.opConfObjList:
468 468 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
469 469 continue
470 470 return opConfObj
471 471 return None
472 472
473 473 def getProcUnitObj(self):
474 474
475 475 return self.procUnitObj
476 476
477 477 def setup(self, id, name, datatype, inputId, parentId=None):
478 478
479 479 #Compatible with old signal chain version
480 480 if datatype==None and name==None:
481 481 raise ValueError, "datatype or name should be defined"
482 482
483 483 if name==None:
484 484 if 'Proc' in datatype:
485 485 name = datatype
486 486 else:
487 487 name = '%sProc' %(datatype)
488 488
489 489 if datatype==None:
490 490 datatype = name.replace('Proc','')
491 491
492 492 self.id = str(id)
493 493 self.name = name
494 494 self.datatype = datatype
495 495 self.inputId = inputId
496 496 self.parentId = parentId
497 497
498 498 self.opConfObjList = []
499 499
500 500 self.addOperation(name='run', optype='self')
501 501
502 502 def removeOperations(self):
503 503
504 504 for obj in self.opConfObjList:
505 505 del obj
506 506
507 507 self.opConfObjList = []
508 508 self.addOperation(name='run')
509 509
510 510 def addParameter(self, **kwargs):
511 511 '''
512 512 Add parameters to "run" operation
513 513 '''
514 514 opObj = self.opConfObjList[0]
515 515
516 516 opObj.addParameter(**kwargs)
517 517
518 518 return opObj
519 519
520 520 def addOperation(self, name, optype='self'):
521 521
522 522 id = self.__getNewId()
523 523 priority = self.__getPriority()
524 524
525 525 opConfObj = OperationConf()
526 526 opConfObj.setup(id, name=name, priority=priority, type=optype)
527 527
528 528 self.opConfObjList.append(opConfObj)
529 529
530 530 return opConfObj
531 531
532 532 def makeXml(self, procUnitElement):
533 533
534 534 upElement = SubElement(procUnitElement, self.ELEMENTNAME)
535 535 upElement.set('id', str(self.id))
536 536 upElement.set('name', self.name)
537 537 upElement.set('datatype', self.datatype)
538 538 upElement.set('inputId', str(self.inputId))
539 539
540 540 for opConfObj in self.opConfObjList:
541 541 opConfObj.makeXml(upElement)
542 542
543 543 def readXml(self, upElement):
544 544
545 545 self.id = upElement.get('id')
546 546 self.name = upElement.get('name')
547 547 self.datatype = upElement.get('datatype')
548 548 self.inputId = upElement.get('inputId')
549 549
550 550 if self.ELEMENTNAME == "ReadUnit":
551 551 self.datatype = self.datatype.replace("Reader", "")
552 552
553 553 if self.ELEMENTNAME == "ProcUnit":
554 554 self.datatype = self.datatype.replace("Proc", "")
555 555
556 556 if self.inputId == 'None':
557 557 self.inputId = '0'
558 558
559 559 self.opConfObjList = []
560 560
561 561 opElementList = upElement.getiterator(OperationConf().getElementName())
562 562
563 563 for opElement in opElementList:
564 564 opConfObj = OperationConf()
565 565 opConfObj.readXml(opElement)
566 566 self.opConfObjList.append(opConfObj)
567 567
568 568 def printattr(self):
569 569
570 570 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
571 571 self.id,
572 572 self.name,
573 573 self.datatype,
574 574 self.inputId)
575 575
576 576 for opConfObj in self.opConfObjList:
577 577 opConfObj.printattr()
578 578
579 579 def createObjects(self):
580 580
581 581 className = eval(self.name)
582 582 procUnitObj = className()
583 583
584 584 for opConfObj in self.opConfObjList:
585 585
586 586 if opConfObj.type == 'self':
587 587 continue
588 588
589 589 opObj = opConfObj.createObject()
590 590
591 591 self.opObjDict[opConfObj.id] = opObj
592 592 procUnitObj.addOperation(opObj, opConfObj.id)
593 593
594 594 self.procUnitObj = procUnitObj
595 595
596 596 return procUnitObj
597 597
598 598 def run(self):
599 599
600 600 finalSts = False
601 601
602 602 for opConfObj in self.opConfObjList:
603 603
604 604 kwargs = {}
605 605 for parmConfObj in opConfObj.getParameterObjList():
606 606 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
607 607 continue
608 608
609 609 kwargs[parmConfObj.name] = parmConfObj.getValue()
610 610
611 611 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
612 612 sts = self.procUnitObj.call(opType = opConfObj.type,
613 613 opName = opConfObj.name,
614 614 opId = opConfObj.id,
615 615 **kwargs)
616 616 finalSts = finalSts or sts
617 617
618 618 return finalSts
619 619
620 620 def close(self):
621 621
622 622 for opConfObj in self.opConfObjList:
623 623 if opConfObj.type == 'self':
624 624 continue
625 625
626 626 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
627 627 opObj.close()
628 628
629 629 self.procUnitObj.close()
630 630
631 631 return
632 632
633 633 class ReadUnitConf(ProcUnitConf):
634 634
635 635 path = None
636 636 startDate = None
637 637 endDate = None
638 638 startTime = None
639 639 endTime = None
640 640
641 641 ELEMENTNAME = 'ReadUnit'
642 642
643 643 def __init__(self):
644 644
645 645 self.id = None
646 646 self.datatype = None
647 647 self.name = None
648 648 self.inputId = None
649 649
650 650 self.parentId = None
651 651
652 652 self.opConfObjList = []
653 653 self.opObjList = []
654 654
655 655 def getElementName(self):
656 656
657 657 return self.ELEMENTNAME
658 658
659 659 def setup(self, id, name, datatype, path, startDate="", endDate="", startTime="", endTime="", parentId=None, **kwargs):
660 660
661 661 #Compatible with old signal chain version
662 662 if datatype==None and name==None:
663 663 raise ValueError, "datatype or name should be defined"
664 664
665 665 if name==None:
666 666 if 'Reader' in datatype:
667 667 name = datatype
668 668 else:
669 669 name = '%sReader' %(datatype)
670 670
671 671 if datatype==None:
672 672 datatype = name.replace('Reader','')
673 673
674 674 self.id = id
675 675 self.name = name
676 676 self.datatype = datatype
677 677
678 678 self.path = path
679 679 self.startDate = startDate
680 680 self.endDate = endDate
681 681 self.startTime = startTime
682 682 self.endTime = endTime
683 683
684 684 self.inputId = '0'
685 685 self.parentId = parentId
686 686
687 687 self.addRunOperation(**kwargs)
688 688
689 689 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
690 690
691 691 #Compatible with old signal chain version
692 692 if datatype==None and name==None:
693 693 raise ValueError, "datatype or name should be defined"
694 694
695 695 if name==None:
696 696 if 'Reader' in datatype:
697 697 name = datatype
698 698 else:
699 699 name = '%sReader' %(datatype)
700 700
701 701 if datatype==None:
702 702 datatype = name.replace('Reader','')
703 703
704 704 self.datatype = datatype
705 705 self.name = name
706 706 self.path = path
707 707 self.startDate = startDate
708 708 self.endDate = endDate
709 709 self.startTime = startTime
710 710 self.endTime = endTime
711 711
712 712 self.inputId = '0'
713 713 self.parentId = parentId
714 714
715 715 self.updateRunOperation(**kwargs)
716 716
717 717 def addRunOperation(self, **kwargs):
718 718
719 719 opObj = self.addOperation(name = 'run', optype = 'self')
720 720
721 721 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
722 722 opObj.addParameter(name='path' , value=self.path, format='str')
723 723 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
724 724 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
725 725 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
726 726 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
727 727
728 728 for key, value in kwargs.items():
729 729 opObj.addParameter(name=key, value=value, format=type(value).__name__)
730 730
731 731 return opObj
732 732
733 733 def updateRunOperation(self, **kwargs):
734 734
735 735 opObj = self.getOperationObj(name = 'run')
736 736 opObj.removeParameters()
737 737
738 738 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
739 739 opObj.addParameter(name='path' , value=self.path, format='str')
740 740 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
741 741 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
742 742 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
743 743 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
744 744
745 745 for key, value in kwargs.items():
746 746 opObj.addParameter(name=key, value=value, format=type(value).__name__)
747 747
748 748 return opObj
749 749
750 750 class Project():
751 751
752 752 id = None
753 753 name = None
754 754 description = None
755 755 # readUnitConfObjList = None
756 756 procUnitConfObjDict = None
757 757
758 758 ELEMENTNAME = 'Project'
759 759
760 760 def __init__(self, control=None, dataq=None):
761 761
762 762 self.id = None
763 763 self.name = None
764 764 self.description = None
765 765
766 766 self.procUnitConfObjDict = {}
767 767
768 768 #global data_q
769 769 #data_q = dataq
770 770
771 771 if control==None:
772 772 control = {'stop':False,'pause':False}
773 773
774 774 self.control = control
775 775
776 776 def __getNewId(self):
777 777
778 778 id = int(self.id)*10 + len(self.procUnitConfObjDict) + 1
779 779
780 780 return str(id)
781 781
782 782 def getElementName(self):
783 783
784 784 return self.ELEMENTNAME
785 785
786 786 def getId(self):
787 787
788 788 return self.id
789 789
790 790 def updateId(self, new_id):
791 791
792 792 self.id = str(new_id)
793 793
794 794 keyList = self.procUnitConfObjDict.keys()
795 795 keyList.sort()
796 796
797 797 n = 1
798 798 newProcUnitConfObjDict = {}
799 799
800 800 for procKey in keyList:
801 801
802 802 procUnitConfObj = self.procUnitConfObjDict[procKey]
803 803 idProcUnit = str(int(self.id)*10 + n)
804 804 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
805 805
806 806 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
807 807 n += 1
808 808
809 809 self.procUnitConfObjDict = newProcUnitConfObjDict
810 810
811 811 def setup(self, id, name, description):
812 812
813 813 self.id = str(id)
814 814 self.name = name
815 815 self.description = description
816 816
817 817 def update(self, name, description):
818 818
819 819 self.name = name
820 820 self.description = description
821 821
822 822 def addReadUnit(self, datatype=None, name=None, **kwargs):
823 823
824 824 idReadUnit = self.__getNewId()
825 825
826 826 readUnitConfObj = ReadUnitConf()
827 827 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
828 828
829 829 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
830 830
831 831 return readUnitConfObj
832 832
833 833 def addProcUnit(self, inputId='0', datatype=None, name=None):
834 834
835 835 idProcUnit = self.__getNewId()
836 836
837 837 procUnitConfObj = ProcUnitConf()
838 838 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
839 839
840 840 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
841 841
842 842 return procUnitConfObj
843 843
844 844 def removeProcUnit(self, id):
845 845
846 846 if id in self.procUnitConfObjDict.keys():
847 847 self.procUnitConfObjDict.pop(id)
848 848
849 849 def getReadUnitId(self):
850 850
851 851 readUnitConfObj = self.getReadUnitObj()
852 852
853 853 return readUnitConfObj.id
854 854
855 855 def getReadUnitObj(self):
856 856
857 857 for obj in self.procUnitConfObjDict.values():
858 858 if obj.getElementName() == "ReadUnit":
859 859 return obj
860 860
861 861 return None
862 862
863 863 def getProcUnitObj(self, id=None, name=None):
864 864
865 865 if id != None:
866 866 return self.procUnitConfObjDict[id]
867 867
868 868 if name != None:
869 869 return self.getProcUnitObjByName(name)
870 870
871 871 return None
872 872
873 873 def getProcUnitObjByName(self, name):
874 874
875 875 for obj in self.procUnitConfObjDict.values():
876 876 if obj.name == name:
877 877 return obj
878 878
879 879 return None
880 880
881 881 def procUnitItems(self):
882 882
883 883 return self.procUnitConfObjDict.items()
884 884
885 885 def makeXml(self):
886 886
887 887 projectElement = Element('Project')
888 888 projectElement.set('id', str(self.id))
889 889 projectElement.set('name', self.name)
890 890 projectElement.set('description', self.description)
891 891
892 892 # for readUnitConfObj in self.readUnitConfObjList:
893 893 # readUnitConfObj.makeXml(projectElement)
894 894
895 895 for procUnitConfObj in self.procUnitConfObjDict.values():
896 896 procUnitConfObj.makeXml(projectElement)
897 897
898 898 self.projectElement = projectElement
899 899
900 900 def writeXml(self, filename):
901 901
902 902 self.makeXml()
903 903
904 904 #print prettify(self.projectElement)
905 905
906 906 ElementTree(self.projectElement).write(filename, method='xml')
907 907
908 908 def readXml(self, filename):
909 909
910 910 #tree = ET.parse(filename)
911 911 self.projectElement = None
912 912 # self.readUnitConfObjList = []
913 913 self.procUnitConfObjDict = {}
914 914
915 915 self.projectElement = ElementTree().parse(filename)
916 916
917 917 self.project = self.projectElement.tag
918 918
919 919 self.id = self.projectElement.get('id')
920 920 self.name = self.projectElement.get('name')
921 921 self.description = self.projectElement.get('description')
922 922
923 923 readUnitElementList = self.projectElement.getiterator(ReadUnitConf().getElementName())
924 924
925 925 for readUnitElement in readUnitElementList:
926 926 readUnitConfObj = ReadUnitConf()
927 927 readUnitConfObj.readXml(readUnitElement)
928 928
929 929 if readUnitConfObj.parentId == None:
930 930 readUnitConfObj.parentId = self.id
931 931
932 932 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
933 933
934 934 procUnitElementList = self.projectElement.getiterator(ProcUnitConf().getElementName())
935 935
936 936 for procUnitElement in procUnitElementList:
937 937 procUnitConfObj = ProcUnitConf()
938 938 procUnitConfObj.readXml(procUnitElement)
939 939
940 940 if procUnitConfObj.parentId == None:
941 941 procUnitConfObj.parentId = self.id
942 942
943 943 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
944 944
945 945 def printattr(self):
946 946
947 947 print "Project[%s]: name = %s, description = %s" %(self.id,
948 948 self.name,
949 949 self.description)
950 950
951 951 # for readUnitConfObj in self.readUnitConfObjList:
952 952 # readUnitConfObj.printattr()
953 953
954 954 for procUnitConfObj in self.procUnitConfObjDict.values():
955 955 procUnitConfObj.printattr()
956 956
957 957 def createObjects(self):
958 958
959 959 # for readUnitConfObj in self.readUnitConfObjList:
960 960 # readUnitConfObj.createObjects()
961 961
962 962 for procUnitConfObj in self.procUnitConfObjDict.values():
963 963 procUnitConfObj.createObjects()
964 964
965 965 def __connect(self, objIN, thisObj):
966 966
967 967 thisObj.setInput(objIN.getOutputObj())
968 968
969 969 def connectObjects(self):
970 970
971 971 for thisPUConfObj in self.procUnitConfObjDict.values():
972 972
973 973 inputId = thisPUConfObj.getInputId()
974 974
975 975 if int(inputId) == 0:
976 976 continue
977 977
978 978 #Get input object
979 979 puConfINObj = self.procUnitConfObjDict[inputId]
980 980 puObjIN = puConfINObj.getProcUnitObj()
981 981
982 982 #Get current object
983 983 thisPUObj = thisPUConfObj.getProcUnitObj()
984 984
985 985 self.__connect(puObjIN, thisPUObj)
986 986
987 987 def run(self):
988 988
989 989 # for readUnitConfObj in self.readUnitConfObjList:
990 990 # readUnitConfObj.run()
991 991 print
992 992 print "*"*40
993 993 print " Starting SIGNAL CHAIN PROCESSING "
994 994 print "*"*40
995 995 print
996 996
997 997 keyList = self.procUnitConfObjDict.keys()
998 998 keyList.sort()
999 999
1000 1000 while(True):
1001 1001
1002 1002 finalSts = False
1003 1003
1004 1004 for procKey in keyList:
1005 1005 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1006 1006
1007 1007 procUnitConfObj = self.procUnitConfObjDict[procKey]
1008 1008 sts = procUnitConfObj.run()
1009 1009 finalSts = finalSts or sts
1010 1010
1011 1011 #If every process unit finished so end process
1012 1012 if not(finalSts):
1013 1013 print "Every process unit have finished"
1014 1014 break
1015 1015
1016 1016 if self.control['pause']:
1017 1017 print "Process suspended"
1018 1018
1019 1019 while True:
1020 1020 sleep(0.1)
1021 1021
1022 1022 if not self.control['pause']:
1023 1023 break
1024 1024
1025 1025 if self.control['stop']:
1026 1026 break
1027 1027 print "Process reinitialized"
1028 1028
1029 1029 if self.control['stop']:
1030 1030 print "Process stopped"
1031 1031 break
1032 1032
1033 1033 #Closing every process
1034 1034 for procKey in keyList:
1035 1035 procUnitConfObj = self.procUnitConfObjDict[procKey]
1036 1036 procUnitConfObj.close()
1037 1037
1038 1038 print "Process finished"
1039 1039
1040 1040 def start(self, filename):
1041 1041
1042 1042 self.writeXml(filename)
1043 1043 self.readXml(filename)
1044 1044
1045 1045 self.createObjects()
1046 1046 self.connectObjects()
1047 1047 self.run()
1048 1048
1049 1049 class ControllerThread(threading.Thread, Project):
1050 1050
1051 1051 def __init__(self, filename):
1052 1052
1053 1053 threading.Thread.__init__(self)
1054 1054 Project.__init__(self)
1055 1055
1056 1056 self.setDaemon(True)
1057 1057
1058 1058 self.filename = filename
1059 1059 self.control = {'stop':False, 'pause':False}
1060 1060
1061 1061 def stop(self):
1062 1062 self.control['stop'] = True
1063 1063
1064 1064 def pause(self):
1065 1065 self.control['pause'] = not(self.control['pause'])
1066 1066
1067 1067 def run(self):
1068 1068 self.control['stop'] = False
1069 1069 self.control['pause'] = False
1070 1070
1071 1071 self.readXml(self.filename)
1072 1072 self.createObjects()
1073 1073 self.connectObjects()
1074 1074 Project.run(self)
1075 1075
1076 1076 if __name__ == '__main__':
1077 1077
1078 1078 desc = "Segundo Test"
1079 1079 filename = "schain.xml"
1080 1080
1081 1081 controllerObj = Project()
1082 1082
1083 1083 controllerObj.setup(id = '191', name='test01', description=desc)
1084 1084
1085 1085 readUnitConfObj = controllerObj.addReadUnit(datatype='Voltage',
1086 1086 path='data/rawdata/',
1087 1087 startDate='2011/01/01',
1088 1088 endDate='2012/12/31',
1089 1089 startTime='00:00:00',
1090 1090 endTime='23:59:59',
1091 1091 online=1,
1092 1092 walk=1)
1093 1093
1094 1094 # opObj00 = readUnitConfObj.addOperation(name='printInfo')
1095 1095
1096 1096 procUnitConfObj0 = controllerObj.addProcUnit(datatype='Voltage', inputId=readUnitConfObj.getId())
1097 1097
1098 1098 opObj10 = procUnitConfObj0.addOperation(name='selectChannels')
1099 1099 opObj10.addParameter(name='channelList', value='3,4,5', format='intlist')
1100 1100
1101 1101 opObj10 = procUnitConfObj0.addOperation(name='selectHeights')
1102 1102 opObj10.addParameter(name='minHei', value='90', format='float')
1103 1103 opObj10.addParameter(name='maxHei', value='180', format='float')
1104 1104
1105 1105 opObj12 = procUnitConfObj0.addOperation(name='CohInt', optype='external')
1106 1106 opObj12.addParameter(name='n', value='10', format='int')
1107 1107
1108 1108 procUnitConfObj1 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj0.getId())
1109 1109 procUnitConfObj1.addParameter(name='nFFTPoints', value='32', format='int')
1110 1110 # procUnitConfObj1.addParameter(name='pairList', value='(0,1),(0,2),(1,2)', format='')
1111 1111
1112 1112
1113 1113 opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
1114 1114 opObj11.addParameter(name='idfigure', value='1', format='int')
1115 1115 opObj11.addParameter(name='wintitle', value='SpectraPlot0', format='str')
1116 1116 opObj11.addParameter(name='zmin', value='40', format='int')
1117 1117 opObj11.addParameter(name='zmax', value='90', format='int')
1118 1118 opObj11.addParameter(name='showprofile', value='1', format='int')
1119 1119
1120 1120 # opObj11 = procUnitConfObj1.addOperation(name='CrossSpectraPlot', optype='external')
1121 1121 # opObj11.addParameter(name='idfigure', value='2', format='int')
1122 1122 # opObj11.addParameter(name='wintitle', value='CrossSpectraPlot', format='str')
1123 1123 # opObj11.addParameter(name='zmin', value='40', format='int')
1124 1124 # opObj11.addParameter(name='zmax', value='90', format='int')
1125 1125
1126 1126
1127 1127 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Voltage', inputId=procUnitConfObj0.getId())
1128 1128 #
1129 1129 # opObj12 = procUnitConfObj2.addOperation(name='CohInt', optype='external')
1130 1130 # opObj12.addParameter(name='n', value='2', format='int')
1131 1131 # opObj12.addParameter(name='overlapping', value='1', format='int')
1132 1132 #
1133 1133 # procUnitConfObj3 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj2.getId())
1134 1134 # procUnitConfObj3.addParameter(name='nFFTPoints', value='32', format='int')
1135 1135 #
1136 1136 # opObj11 = procUnitConfObj3.addOperation(name='SpectraPlot', optype='external')
1137 1137 # opObj11.addParameter(name='idfigure', value='2', format='int')
1138 1138 # opObj11.addParameter(name='wintitle', value='SpectraPlot1', format='str')
1139 1139 # opObj11.addParameter(name='zmin', value='40', format='int')
1140 1140 # opObj11.addParameter(name='zmax', value='90', format='int')
1141 1141 # opObj11.addParameter(name='showprofile', value='1', format='int')
1142 1142
1143 1143 # opObj11 = procUnitConfObj1.addOperation(name='RTIPlot', optype='external')
1144 1144 # opObj11.addParameter(name='idfigure', value='10', format='int')
1145 1145 # opObj11.addParameter(name='wintitle', value='RTI', format='str')
1146 1146 ## opObj11.addParameter(name='xmin', value='21', format='float')
1147 1147 ## opObj11.addParameter(name='xmax', value='22', format='float')
1148 1148 # opObj11.addParameter(name='zmin', value='40', format='int')
1149 1149 # opObj11.addParameter(name='zmax', value='90', format='int')
1150 1150 # opObj11.addParameter(name='showprofile', value='1', format='int')
1151 1151 # opObj11.addParameter(name='timerange', value=str(60), format='int')
1152 1152
1153 1153 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
1154 1154 # opObj10.addParameter(name='channelList', value='0,2,4,6', format='intlist')
1155 1155 #
1156 1156 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
1157 1157 # opObj12.addParameter(name='n', value='2', format='int')
1158 1158 #
1159 1159 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
1160 1160 # opObj11.addParameter(name='idfigure', value='2', format='int')
1161 1161 # opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
1162 1162 # opObj11.addParameter(name='zmin', value='70', format='int')
1163 1163 # opObj11.addParameter(name='zmax', value='90', format='int')
1164 1164 #
1165 1165 # opObj10 = procUnitConfObj1.addOperation(name='selectChannels')
1166 1166 # opObj10.addParameter(name='channelList', value='2,6', format='intlist')
1167 1167 #
1168 1168 # opObj12 = procUnitConfObj1.addOperation(name='IncohInt', optype='external')
1169 1169 # opObj12.addParameter(name='n', value='2', format='int')
1170 1170 #
1171 1171 # opObj11 = procUnitConfObj1.addOperation(name='SpectraPlot', optype='external')
1172 1172 # opObj11.addParameter(name='idfigure', value='3', format='int')
1173 1173 # opObj11.addParameter(name='wintitle', value='SpectraPlot10', format='str')
1174 1174 # opObj11.addParameter(name='zmin', value='70', format='int')
1175 1175 # opObj11.addParameter(name='zmax', value='90', format='int')
1176 1176
1177 1177
1178 1178 # opObj12 = procUnitConfObj1.addOperation(name='decoder')
1179 1179 # opObj12.addParameter(name='ncode', value='2', format='int')
1180 1180 # opObj12.addParameter(name='nbauds', value='8', format='int')
1181 1181 # opObj12.addParameter(name='code0', value='001110011', format='int')
1182 1182 # opObj12.addParameter(name='code1', value='001110011', format='int')
1183 1183
1184 1184
1185 1185
1186 1186 # procUnitConfObj2 = controllerObj.addProcUnit(datatype='Spectra', inputId=procUnitConfObj1.getId())
1187 1187 #
1188 1188 # opObj21 = procUnitConfObj2.addOperation(name='IncohInt', optype='external')
1189 1189 # opObj21.addParameter(name='n', value='2', format='int')
1190 1190 #
1191 1191 # opObj11 = procUnitConfObj2.addOperation(name='SpectraPlot', optype='external')
1192 1192 # opObj11.addParameter(name='idfigure', value='4', format='int')
1193 1193 # opObj11.addParameter(name='wintitle', value='SpectraPlot OBJ 2', format='str')
1194 1194 # opObj11.addParameter(name='zmin', value='70', format='int')
1195 1195 # opObj11.addParameter(name='zmax', value='90', format='int')
1196 1196
1197 1197 print "Escribiendo el archivo XML"
1198 1198
1199 1199 controllerObj.writeXml(filename)
1200 1200
1201 1201 print "Leyendo el archivo XML"
1202 1202 controllerObj.readXml(filename)
1203 1203 #controllerObj.printattr()
1204 1204
1205 1205 controllerObj.createObjects()
1206 1206 controllerObj.connectObjects()
1207 1207 controllerObj.run()
1208 1208
1209 1209 No newline at end of file
Show file before | Show file after | Hide comments
1 NO CONTENT: modified file
The requested commit or file is too big and content was truncated. Show full diff
Show file before | Show file after | Hide comments
@@ -1,1451 +1,1465
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import time, datetime
13 13 #import h5py
14 14 import traceback
15 15
16 16 try:
17 17 from gevent import sleep
18 18 except:
19 19 from time import sleep
20 20
21 21 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
22 22
23 23 LOCALTIME = True
24 24
25 25 def isNumber(cad):
26 26 """
27 27 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
28 28
29 29 Excepciones:
30 30 Si un determinado string no puede ser convertido a numero
31 31 Input:
32 32 str, string al cual se le analiza para determinar si convertible a un numero o no
33 33
34 34 Return:
35 35 True : si el string es uno numerico
36 36 False : no es un string numerico
37 37 """
38 38 try:
39 39 float( cad )
40 40 return True
41 41 except:
42 42 return False
43 43
44 44 def isThisFileinRange(filename, startUTSeconds, endUTSeconds):
45 45 """
46 46 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
47 47
48 48 Inputs:
49 49 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
50 50
51 51 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
52 52 segundos contados desde 01/01/1970.
53 53 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55
56 56 Return:
57 57 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
58 58 fecha especificado, de lo contrario retorna False.
59 59
60 60 Excepciones:
61 61 Si el archivo no existe o no puede ser abierto
62 62 Si la cabecera no puede ser leida.
63 63
64 64 """
65 65 basicHeaderObj = BasicHeader(LOCALTIME)
66 66
67 67 try:
68 68 fp = open(filename,'rb')
69 69 except IOError:
70 70 traceback.print_exc()
71 71 raise IOError, "The file %s can't be opened" %(filename)
72 72
73 73 sts = basicHeaderObj.read(fp)
74 74 fp.close()
75 75
76 76 if not(sts):
77 77 print "Skipping the file %s because it has not a valid header" %(filename)
78 78 return 0
79 79
80 80 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
81 81 return 0
82 82
83 83 return 1
84 84
85 85 def isFileinThisTime(filename, startTime, endTime):
86 86 """
87 87 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
88 88
89 89 Inputs:
90 90 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
91 91
92 92 startTime : tiempo inicial del rango seleccionado en formato datetime.time
93 93
94 94 endTime : tiempo final del rango seleccionado en formato datetime.time
95 95
96 96 Return:
97 97 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
98 98 fecha especificado, de lo contrario retorna False.
99 99
100 100 Excepciones:
101 101 Si el archivo no existe o no puede ser abierto
102 102 Si la cabecera no puede ser leida.
103 103
104 104 """
105 105
106 106
107 107 try:
108 108 fp = open(filename,'rb')
109 109 except IOError:
110 110 traceback.print_exc()
111 111 raise IOError, "The file %s can't be opened" %(filename)
112 112
113 113 basicHeaderObj = BasicHeader(LOCALTIME)
114 114 sts = basicHeaderObj.read(fp)
115 115 fp.close()
116 116
117 117 thisDatetime = basicHeaderObj.datatime
118 118 thisTime = thisDatetime.time()
119 119
120 120 if not(sts):
121 121 print "Skipping the file %s because it has not a valid header" %(filename)
122 122 return None
123 123
124 124 if not ((startTime <= thisTime) and (endTime > thisTime)):
125 125 return None
126 126
127 127 return thisDatetime
128 128
129 129 def getFileFromSet(path, ext, set):
130 130 validFilelist = []
131 131 fileList = os.listdir(path)
132 132
133 133 # 0 1234 567 89A BCDE
134 134 # H YYYY DDD SSS .ext
135 135
136 136 for thisFile in fileList:
137 137 try:
138 138 year = int(thisFile[1:5])
139 139 doy = int(thisFile[5:8])
140 140 except:
141 141 continue
142 142
143 143 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
144 144 continue
145 145
146 146 validFilelist.append(thisFile)
147 147
148 148 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
149 149
150 150 if len(myfile)!= 0:
151 151 return myfile[0]
152 152 else:
153 153 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
154 154 print 'the filename %s does not exist'%filename
155 155 print '...going to the last file: '
156 156
157 157 if validFilelist:
158 158 validFilelist = sorted( validFilelist, key=str.lower )
159 159 return validFilelist[-1]
160 160
161 161 return None
162 162
163 163 def getlastFileFromPath(path, ext):
164 164 """
165 165 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
166 166 al final de la depuracion devuelve el ultimo file de la lista que quedo.
167 167
168 168 Input:
169 169 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
170 170 ext : extension de los files contenidos en una carpeta
171 171
172 172 Return:
173 173 El ultimo file de una determinada carpeta, no se considera el path.
174 174 """
175 175 validFilelist = []
176 176 fileList = os.listdir(path)
177 177
178 178 # 0 1234 567 89A BCDE
179 179 # H YYYY DDD SSS .ext
180 180
181 181 for thisFile in fileList:
182 182
183 183 year = thisFile[1:5]
184 184 if not isNumber(year):
185 185 continue
186 186
187 187 doy = thisFile[5:8]
188 188 if not isNumber(doy):
189 189 continue
190 190
191 191 year = int(year)
192 192 doy = int(doy)
193 193
194 194 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
195 195 continue
196 196
197 197 validFilelist.append(thisFile)
198 198
199 199 if validFilelist:
200 200 validFilelist = sorted( validFilelist, key=str.lower )
201 201 return validFilelist[-1]
202 202
203 203 return None
204 204
205 205 def checkForRealPath(path, foldercounter, year, doy, set, ext):
206 206 """
207 207 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
208 208 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
209 209 el path exacto de un determinado file.
210 210
211 211 Example :
212 212 nombre correcto del file es .../.../D2009307/P2009307367.ext
213 213
214 214 Entonces la funcion prueba con las siguientes combinaciones
215 215 .../.../y2009307367.ext
216 216 .../.../Y2009307367.ext
217 217 .../.../x2009307/y2009307367.ext
218 218 .../.../x2009307/Y2009307367.ext
219 219 .../.../X2009307/y2009307367.ext
220 220 .../.../X2009307/Y2009307367.ext
221 221 siendo para este caso, la ultima combinacion de letras, identica al file buscado
222 222
223 223 Return:
224 224 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
225 225 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
226 226 para el filename
227 227 """
228 228 fullfilename = None
229 229 find_flag = False
230 230 filename = None
231 231
232 232 prefixDirList = [None,'d','D']
233 233 if ext.lower() == ".r": #voltage
234 234 prefixFileList = ['d','D']
235 235 elif ext.lower() == ".pdata": #spectra
236 236 prefixFileList = ['p','P']
237 237 else:
238 238 return None, filename
239 239
240 240 #barrido por las combinaciones posibles
241 241 for prefixDir in prefixDirList:
242 242 thispath = path
243 243 if prefixDir != None:
244 244 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
245 245 if foldercounter == 0:
246 246 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
247 247 else:
248 248 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
249 249 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
250 250 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
251 251 fullfilename = os.path.join( thispath, filename ) #formo el path completo
252 252
253 253 if os.path.exists( fullfilename ): #verifico que exista
254 254 find_flag = True
255 255 break
256 256 if find_flag:
257 257 break
258 258
259 259 if not(find_flag):
260 260 return None, filename
261 261
262 262 return fullfilename, filename
263 263
264 264 def isRadarFolder(folder):
265 265 try:
266 266 year = int(folder[1:5])
267 267 doy = int(folder[5:8])
268 268 except:
269 269 return 0
270 270
271 271 return 1
272 272
273 273 def isRadarFile(file):
274 274 try:
275 275 year = int(file[1:5])
276 276 doy = int(file[5:8])
277 277 set = int(file[8:11])
278 278 except:
279 279 return 0
280 280
281 281 return 1
282 282
283 283 def getDateFromRadarFile(file):
284 284 try:
285 285 year = int(file[1:5])
286 286 doy = int(file[5:8])
287 287 set = int(file[8:11])
288 288 except:
289 289 return None
290 290
291 291 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
292 292 return thisDate
293 293
294 294 class JRODataIO:
295 295
296 296 c = 3E8
297 297
298 298 isConfig = False
299 299
300 300 basicHeaderObj = None
301 301
302 302 systemHeaderObj = None
303 303
304 304 radarControllerHeaderObj = None
305 305
306 306 processingHeaderObj = None
307 307
308 308 online = 0
309 309
310 310 dtype = None
311 311
312 312 pathList = []
313 313
314 314 filenameList = []
315 315
316 316 filename = None
317 317
318 318 ext = None
319 319
320 320 flagIsNewFile = 1
321 321
322 322 flagDiscontinuousBlock = 0
323 323
324 324 flagIsNewBlock = 0
325 325
326 326 fp = None
327 327
328 328 firstHeaderSize = 0
329 329
330 330 basicHeaderSize = 24
331 331
332 332 versionFile = 1103
333 333
334 334 fileSize = None
335 335
336 336 # ippSeconds = None
337 337
338 338 fileSizeByHeader = None
339 339
340 340 fileIndex = None
341 341
342 342 profileIndex = None
343 343
344 344 blockIndex = None
345 345
346 346 nTotalBlocks = None
347 347
348 348 maxTimeStep = 30
349 349
350 350 lastUTTime = None
351 351
352 352 datablock = None
353 353
354 354 dataOut = None
355 355
356 356 blocksize = None
357 357
358 358 getByBlock = False
359 359
360 360 def __init__(self):
361 361
362 362 raise ValueError, "Not implemented"
363 363
364 364 def run(self):
365 365
366 366 raise ValueError, "Not implemented"
367 367
368 368 class JRODataReader(JRODataIO):
369 369
370 370 nReadBlocks = 0
371 371
372 372 delay = 10 #number of seconds waiting a new file
373 373
374 374 nTries = 3 #quantity tries
375 375
376 376 nFiles = 3 #number of files for searching
377 377
378 378 path = None
379 379
380 380 foldercounter = 0
381 381
382 382 flagNoMoreFiles = 0
383 383
384 384 datetimeList = []
385 385
386 386 __isFirstTimeOnline = 1
387 387
388 388 __printInfo = True
389 389
390 390 profileIndex = None
391 391
392 392 nTxs = 1
393 393
394 394 txIndex = None
395 395
396 396 def __init__(self):
397 397
398 398 """
399 399
400 400 """
401 401
402 402 # raise NotImplementedError, "This method has not been implemented"
403 403
404 404
405 405 def createObjByDefault(self):
406 406 """
407 407
408 408 """
409 409 raise NotImplementedError, "This method has not been implemented"
410 410
411 411 def getBlockDimension(self):
412 412
413 413 raise NotImplementedError, "No implemented"
414 414
415 415 def __searchFilesOffLine(self,
416 416 path,
417 417 startDate=None,
418 418 endDate=None,
419 419 startTime=datetime.time(0,0,0),
420 420 endTime=datetime.time(23,59,59),
421 421 set=None,
422 422 expLabel='',
423 423 ext='.r',
424 424 walk=True):
425 425
426 426 self.filenameList = []
427 427 self.datetimeList = []
428 428
429 429 pathList = []
430 430
431 431 if not walk:
432 432 #pathList.append(path)
433 433 multi_path = path.split(',')
434 434 for single_path in multi_path:
435
436 if not os.path.isdir(single_path):
437 continue
438
435 439 pathList.append(single_path)
436 440
437 441 else:
438 442 #dirList = []
439 443 multi_path = path.split(',')
440 444 for single_path in multi_path:
445
446 if not os.path.isdir(single_path):
447 continue
448
441 449 dirList = []
442 450 for thisPath in os.listdir(single_path):
443 451 if not os.path.isdir(os.path.join(single_path,thisPath)):
444 452 continue
445 453 if not isRadarFolder(thisPath):
446 454 continue
447 455
448 456 dirList.append(thisPath)
449 457
450 458 if not(dirList):
451 459 return None, None
452 460
453 461 if startDate and endDate:
454 462 thisDate = startDate
455 463
456 464 while(thisDate <= endDate):
457 465 year = thisDate.timetuple().tm_year
458 466 doy = thisDate.timetuple().tm_yday
459 467
460 468 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
461 469 if len(matchlist) == 0:
462 470 thisDate += datetime.timedelta(1)
463 471 continue
464 472 for match in matchlist:
465 473 pathList.append(os.path.join(single_path,match,expLabel))
466 474
467 475 thisDate += datetime.timedelta(1)
468 476 else:
469 477 for thiDir in dirList:
470 478 pathList.append(os.path.join(single_path,thiDir,expLabel))
471 479
472 480 if pathList == []:
473 481 print "Any folder was found for the date range: %s-%s" %(startDate, endDate)
474 482 return None, None
475 483
476 484 print "%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate)
477 485
478 486 filenameList = []
479 487 datetimeList = []
480 488 pathDict = {}
481 489 filenameList_to_sort = []
482 490
483 491 for i in range(len(pathList)):
484 492
485 493 thisPath = pathList[i]
486 494
487 495 fileList = glob.glob1(thisPath, "*%s" %ext)
488 496 if len(fileList) < 1:
489 497 continue
490 498 fileList.sort()
491 499 pathDict.setdefault(fileList[0])
492 500 pathDict[fileList[0]] = i
493 501 filenameList_to_sort.append(fileList[0])
494 502
495 503 filenameList_to_sort.sort()
496 504
497 505 for file in filenameList_to_sort:
498 506 thisPath = pathList[pathDict[file]]
499 507
500 508 fileList = glob.glob1(thisPath, "*%s" %ext)
501 509 fileList.sort()
502 510
503 511 for file in fileList:
504 512
505 513 filename = os.path.join(thisPath,file)
506 514 thisDatetime = isFileinThisTime(filename, startTime, endTime)
507 515
508 516 if not(thisDatetime):
509 517 continue
510 518
511 519 filenameList.append(filename)
512 520 datetimeList.append(thisDatetime)
513 521
514 522 if not(filenameList):
515 523 print "Any file was found for the time range %s - %s" %(startTime, endTime)
516 524 return None, None
517 525
518 526 print "%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime)
519 527 print
520 528
521 529 for i in range(len(filenameList)):
522 530 print "%s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
523 531
524 532 self.filenameList = filenameList
525 533 self.datetimeList = datetimeList
526 534
527 535 return pathList, filenameList
528 536
529 537 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
530 538
531 539 """
532 540 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
533 541 devuelve el archivo encontrado ademas de otros datos.
534 542
535 543 Input:
536 544 path : carpeta donde estan contenidos los files que contiene data
537 545
538 546 expLabel : Nombre del subexperimento (subfolder)
539 547
540 548 ext : extension de los files
541 549
542 550 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
543 551
544 552 Return:
545 553 directory : eL directorio donde esta el file encontrado
546 554 filename : el ultimo file de una determinada carpeta
547 555 year : el anho
548 556 doy : el numero de dia del anho
549 557 set : el set del archivo
550 558
551 559
552 560 """
553 561 dirList = []
554 562
555 563 if not walk:
556 564 fullpath = path
557 565 foldercounter = 0
558 566 else:
559 567 #Filtra solo los directorios
560 568 for thisPath in os.listdir(path):
561 569 if not os.path.isdir(os.path.join(path,thisPath)):
562 570 continue
563 571 if not isRadarFolder(thisPath):
564 572 continue
565 573
566 574 dirList.append(thisPath)
567 575
568 576 if not(dirList):
569 577 return None, None, None, None, None, None
570 578
571 579 dirList = sorted( dirList, key=str.lower )
572 580
573 581 doypath = dirList[-1]
574 582 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
575 583 fullpath = os.path.join(path, doypath, expLabel)
576 584
577 585
578 586 print "%s folder was found: " %(fullpath )
579 587
580 588 if set == None:
581 589 filename = getlastFileFromPath(fullpath, ext)
582 590 else:
583 591 filename = getFileFromSet(fullpath, ext, set)
584 592
585 593 if not(filename):
586 594 return None, None, None, None, None, None
587 595
588 596 print "%s file was found" %(filename)
589 597
590 598 if not(self.__verifyFile(os.path.join(fullpath, filename))):
591 599 return None, None, None, None, None, None
592 600
593 601 year = int( filename[1:5] )
594 602 doy = int( filename[5:8] )
595 603 set = int( filename[8:11] )
596 604
597 605 return fullpath, foldercounter, filename, year, doy, set
598 606
599 607 def __setNextFileOffline(self):
600 608
601 609 idFile = self.fileIndex
602 610
603 611 while (True):
604 612 idFile += 1
605 613 if not(idFile < len(self.filenameList)):
606 614 self.flagNoMoreFiles = 1
607 615 # print "[Reading] No more Files"
608 616 return 0
609 617
610 618 filename = self.filenameList[idFile]
611 619
612 620 if not(self.__verifyFile(filename)):
613 621 continue
614 622
615 623 fileSize = os.path.getsize(filename)
616 624 fp = open(filename,'rb')
617 625 break
618 626
619 627 self.flagIsNewFile = 1
620 628 self.fileIndex = idFile
621 629 self.filename = filename
622 630 self.fileSize = fileSize
623 631 self.fp = fp
624 632
625 633 # print "[Reading] Setting the file: %s"%self.filename
626 634
627 635 return 1
628 636
629 637 def __setNextFileOnline(self):
630 638 """
631 639 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
632 640 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
633 641 siguientes.
634 642
635 643 Affected:
636 644 self.flagIsNewFile
637 645 self.filename
638 646 self.fileSize
639 647 self.fp
640 648 self.set
641 649 self.flagNoMoreFiles
642 650
643 651 Return:
644 652 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
645 653 1 : si el file fue abierto con exito y esta listo a ser leido
646 654
647 655 Excepciones:
648 656 Si un determinado file no puede ser abierto
649 657 """
650 658 nFiles = 0
651 659 fileOk_flag = False
652 660 firstTime_flag = True
653 661
654 662 self.set += 1
655 663
656 664 if self.set > 999:
657 665 self.set = 0
658 666 self.foldercounter += 1
659 667
660 668 #busca el 1er file disponible
661 669 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
662 670 if fullfilename:
663 671 if self.__verifyFile(fullfilename, False):
664 672 fileOk_flag = True
665 673
666 674 #si no encuentra un file entonces espera y vuelve a buscar
667 675 if not(fileOk_flag):
668 676 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
669 677
670 678 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
671 679 tries = self.nTries
672 680 else:
673 681 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
674 682
675 683 for nTries in range( tries ):
676 684 if firstTime_flag:
677 685 print "\t[Reading] Waiting %0.2f sec for the file \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
678 686 sleep( self.delay )
679 687 else:
680 688 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
681 689
682 690 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
683 691 if fullfilename:
684 692 if self.__verifyFile(fullfilename):
685 693 fileOk_flag = True
686 694 break
687 695
688 696 if fileOk_flag:
689 697 break
690 698
691 699 firstTime_flag = False
692 700
693 701 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
694 702 self.set += 1
695 703
696 704 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
697 705 self.set = 0
698 706 self.doy += 1
699 707 self.foldercounter = 0
700 708
701 709 if fileOk_flag:
702 710 self.fileSize = os.path.getsize( fullfilename )
703 711 self.filename = fullfilename
704 712 self.flagIsNewFile = 1
705 713 if self.fp != None: self.fp.close()
706 714 self.fp = open(fullfilename, 'rb')
707 715 self.flagNoMoreFiles = 0
708 716 # print '[Reading] Setting the file: %s' % fullfilename
709 717 else:
710 718 self.fileSize = 0
711 719 self.filename = None
712 720 self.flagIsNewFile = 0
713 721 self.fp = None
714 722 self.flagNoMoreFiles = 1
715 723 # print '[Reading] No more files to read'
716 724
717 725 return fileOk_flag
718 726
719 727 def setNextFile(self):
720 728 if self.fp != None:
721 729 self.fp.close()
722 730
723 731 if self.online:
724 732 newFile = self.__setNextFileOnline()
725 733 else:
726 734 newFile = self.__setNextFileOffline()
727 735
728 736 if not(newFile):
729 737 print '[Reading] No more files to read'
730 738 return 0
731 739
732 740 print '[Reading] Setting the file: %s' % self.filename
733 741
734 742 self.__readFirstHeader()
735 743 self.nReadBlocks = 0
736 744 return 1
737 745
738 746 def __waitNewBlock(self):
739 747 """
740 748 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
741 749
742 750 Si el modo de lectura es OffLine siempre retorn 0
743 751 """
744 752 if not self.online:
745 753 return 0
746 754
747 755 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
748 756 return 0
749 757
750 758 currentPointer = self.fp.tell()
751 759
752 760 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
753 761
754 762 for nTries in range( self.nTries ):
755 763
756 764 self.fp.close()
757 765 self.fp = open( self.filename, 'rb' )
758 766 self.fp.seek( currentPointer )
759 767
760 768 self.fileSize = os.path.getsize( self.filename )
761 769 currentSize = self.fileSize - currentPointer
762 770
763 771 if ( currentSize >= neededSize ):
764 772 self.basicHeaderObj.read(self.fp)
765 773 return 1
766 774
767 775 if self.fileSize == self.fileSizeByHeader:
768 776 # self.flagEoF = True
769 777 return 0
770 778
771 779 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
772 780 sleep( self.delay )
773 781
774 782
775 783 return 0
776 784
777 785 def waitDataBlock(self,pointer_location):
778 786
779 787 currentPointer = pointer_location
780 788
781 789 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
782 790
783 791 for nTries in range( self.nTries ):
784 792 self.fp.close()
785 793 self.fp = open( self.filename, 'rb' )
786 794 self.fp.seek( currentPointer )
787 795
788 796 self.fileSize = os.path.getsize( self.filename )
789 797 currentSize = self.fileSize - currentPointer
790 798
791 799 if ( currentSize >= neededSize ):
792 800 return 1
793 801
794 802 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
795 803 sleep( self.delay )
796 804
797 805 return 0
798 806
799 807 def __jumpToLastBlock(self):
800 808
801 809 if not(self.__isFirstTimeOnline):
802 810 return
803 811
804 812 csize = self.fileSize - self.fp.tell()
805 813 blocksize = self.processingHeaderObj.blockSize
806 814
807 815 #salta el primer bloque de datos
808 816 if csize > self.processingHeaderObj.blockSize:
809 817 self.fp.seek(self.fp.tell() + blocksize)
810 818 else:
811 819 return
812 820
813 821 csize = self.fileSize - self.fp.tell()
814 822 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
815 823 while True:
816 824
817 825 if self.fp.tell()<self.fileSize:
818 826 self.fp.seek(self.fp.tell() + neededsize)
819 827 else:
820 828 self.fp.seek(self.fp.tell() - neededsize)
821 829 break
822 830
823 831 # csize = self.fileSize - self.fp.tell()
824 832 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
825 833 # factor = int(csize/neededsize)
826 834 # if factor > 0:
827 835 # self.fp.seek(self.fp.tell() + factor*neededsize)
828 836
829 837 self.flagIsNewFile = 0
830 838 self.__isFirstTimeOnline = 0
831 839
832 840 def __setNewBlock(self):
833 841
834 842 if self.fp == None:
835 843 return 0
836 844
837 845 if self.online:
838 846 self.__jumpToLastBlock()
839 847
840 848 if self.flagIsNewFile:
841 849 return 1
842 850
843 851 self.lastUTTime = self.basicHeaderObj.utc
844 852 currentSize = self.fileSize - self.fp.tell()
845 853 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
846 854
847 855 if (currentSize >= neededSize):
848 856 self.basicHeaderObj.read(self.fp)
849 857 return 1
850 858
851 859 if self.__waitNewBlock():
852 860 return 1
853 861
854 862 if not(self.setNextFile()):
855 863 return 0
856 864
857 865 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
858 866
859 867 self.flagDiscontinuousBlock = 0
860 868
861 869 if deltaTime > self.maxTimeStep:
862 870 self.flagDiscontinuousBlock = 1
863 871
864 872 return 1
865 873
866 874 def readNextBlock(self):
867 875 if not(self.__setNewBlock()):
868 876 return 0
869 877
870 878 if not(self.readBlock()):
871 879 return 0
872 880
873 881 return 1
874 882
875 883 def __readFirstHeader(self):
876 884
877 885 self.basicHeaderObj.read(self.fp)
878 886 self.systemHeaderObj.read(self.fp)
879 887 self.radarControllerHeaderObj.read(self.fp)
880 888 self.processingHeaderObj.read(self.fp)
881 889
882 890 self.firstHeaderSize = self.basicHeaderObj.size
883 891
884 892 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
885 893 if datatype == 0:
886 894 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
887 895 elif datatype == 1:
888 896 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
889 897 elif datatype == 2:
890 898 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
891 899 elif datatype == 3:
892 900 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
893 901 elif datatype == 4:
894 902 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
895 903 elif datatype == 5:
896 904 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
897 905 else:
898 906 raise ValueError, 'Data type was not defined'
899 907
900 908 self.dtype = datatype_str
901 909 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
902 910 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
903 911 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
904 912 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
905 913 self.getBlockDimension()
906 914
907 915 def __verifyFile(self, filename, msgFlag=True):
908 916 msg = None
909 917 try:
910 918 fp = open(filename, 'rb')
911 919 currentPosition = fp.tell()
912 920 except IOError:
913 921 traceback.print_exc()
914 922 if msgFlag:
915 923 print "[Reading] The file %s can't be opened" % (filename)
916 924 return False
917 925
918 926 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
919 927
920 928 if neededSize == 0:
921 929 basicHeaderObj = BasicHeader(LOCALTIME)
922 930 systemHeaderObj = SystemHeader()
923 931 radarControllerHeaderObj = RadarControllerHeader()
924 932 processingHeaderObj = ProcessingHeader()
925 933
926 934 try:
927 935 if not( basicHeaderObj.read(fp) ): raise IOError
928 936 if not( systemHeaderObj.read(fp) ): raise IOError
929 937 if not( radarControllerHeaderObj.read(fp) ): raise IOError
930 938 if not( processingHeaderObj.read(fp) ): raise IOError
931 939 # data_type = int(numpy.log2((processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
932 940
933 941 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
934 942
935 943 except IOError:
936 944 traceback.print_exc()
937 945 if msgFlag:
938 946 print "[Reading] The file %s is empty or it hasn't enough data" % filename
939 947
940 948 fp.close()
941 949 return False
942 950 else:
943 951 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
944 952
945 953 fp.close()
946 954 fileSize = os.path.getsize(filename)
947 955 currentSize = fileSize - currentPosition
948 956 if currentSize < neededSize:
949 957 if msgFlag and (msg != None):
950 958 print msg #print"\tSkipping the file %s due to it hasn't enough data" %filename
951 959 return False
952 960
953 961 return True
954 962
955 963 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True):
956 964
957 965 dateList = []
958 966 pathList = []
959 967
960 968 if not walk:
961 969 #pathList.append(path)
962 970 multi_path = path.split(',')
963 971 for single_path in multi_path:
964 972
973 if not os.path.isdir(single_path):
974 continue
975
965 976 ok = False
966 977 fileList = glob.glob1(single_path, "*"+ext)
967 978
968 979 for thisFile in fileList:
969 980
970 981 if not os.path.isfile(os.path.join(single_path, thisFile)):
971 982 continue
972 983
973 984 if not isRadarFile(thisFile):
974 985 continue
975 986
976 987 ok = True
977 988 thisDate = getDateFromRadarFile(thisFile)
978 989
979 990 if thisDate not in dateList:
980 991 dateList.append(thisDate)
981 992
982 993 if ok:
983 994 pathList.append(single_path)
984 995
985 996 return dateList
986 997
987 998 multi_path = path.split(',')
988 999 for single_path in multi_path:
989 1000
1001 if not os.path.isdir(single_path):
1002 continue
1003
990 1004 dirList = []
991 1005
992 1006 for thisPath in os.listdir(single_path):
993 1007
994 1008 if not os.path.isdir(os.path.join(single_path,thisPath)):
995 1009 continue
996 1010
997 1011 if not isRadarFolder(thisPath):
998 1012 continue
999 1013
1000 1014 dirList.append(thisPath)
1001 1015
1002 1016 if not dirList:
1003 1017 return dateList
1004 1018
1005 1019 if startDate and endDate:
1006 1020 thisDate = startDate
1007 1021
1008 1022 while(thisDate <= endDate):
1009 1023 year = thisDate.timetuple().tm_year
1010 1024 doy = thisDate.timetuple().tm_yday
1011 1025
1012 1026 matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
1013 1027 if len(matchlist) == 0:
1014 1028 thisDate += datetime.timedelta(1)
1015 1029 continue
1016 1030
1017 1031 for match in matchlist:
1018 1032 pathList.append(os.path.join(single_path,match,expLabel))
1019 1033 dateList.append(thisDate)
1020 1034
1021 1035 thisDate += datetime.timedelta(1)
1022 1036 else:
1023 1037 for thisDir in dirList:
1024 1038 year = int(thisDir[1:5])
1025 1039 doy = int(thisDir[5:8])
1026 1040 thisDate = datetime.date(year,1,1) + datetime.timedelta(doy-1)
1027 1041
1028 1042 pathList.append(os.path.join(single_path,thisDir,expLabel))
1029 1043 dateList.append(thisDate)
1030 1044
1031 1045 return dateList
1032 1046
1033 1047
1034 1048 def setup(self,
1035 1049 path=None,
1036 1050 startDate=None,
1037 1051 endDate=None,
1038 1052 startTime=datetime.time(0,0,0),
1039 1053 endTime=datetime.time(23,59,59),
1040 1054 set=None,
1041 1055 expLabel = "",
1042 1056 ext = None,
1043 1057 online = False,
1044 1058 delay = 60,
1045 1059 walk = True,
1046 1060 getblock = False,
1047 1061 nTxs = 1):
1048 1062
1049 1063 if path == None:
1050 1064 raise ValueError, "[Reading] The path is not valid"
1051 1065
1052 1066 if ext == None:
1053 1067 ext = self.ext
1054 1068
1055 1069 if online:
1056 1070 print "[Reading] Searching files in online mode..."
1057 1071
1058 1072 for nTries in range( self.nTries ):
1059 1073 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1060 1074
1061 1075 if fullpath:
1062 1076 break
1063 1077
1064 1078 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1065 1079 sleep( self.delay )
1066 1080
1067 1081 if not(fullpath):
1068 1082 print "[Reading] There 'isn't any valid file in %s" % path
1069 1083 return None
1070 1084
1071 1085 self.year = year
1072 1086 self.doy = doy
1073 1087 self.set = set - 1
1074 1088 self.path = path
1075 1089 self.foldercounter = foldercounter
1076 1090 last_set = None
1077 1091
1078 1092 else:
1079 1093 print "[Reading] Searching files in offline mode ..."
1080 1094 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1081 1095 startTime=startTime, endTime=endTime,
1082 1096 set=set, expLabel=expLabel, ext=ext,
1083 1097 walk=walk)
1084 1098
1085 1099 if not(pathList):
1086 1100 print "[Reading] No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
1087 1101 datetime.datetime.combine(startDate,startTime).ctime(),
1088 1102 datetime.datetime.combine(endDate,endTime).ctime())
1089 1103
1090 1104 sys.exit(-1)
1091 1105
1092 1106
1093 1107 self.fileIndex = -1
1094 1108 self.pathList = pathList
1095 1109 self.filenameList = filenameList
1096 1110 file_name = os.path.basename(filenameList[-1])
1097 1111 basename, ext = os.path.splitext(file_name)
1098 1112 last_set = int(basename[-3:])
1099 1113
1100 1114 self.online = online
1101 1115 self.delay = delay
1102 1116 ext = ext.lower()
1103 1117 self.ext = ext
1104 1118 self.getByBlock = getblock
1105 1119 self.nTxs = int(nTxs)
1106 1120
1107 1121 if not(self.setNextFile()):
1108 1122 if (startDate!=None) and (endDate!=None):
1109 1123 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1110 1124 elif startDate != None:
1111 1125 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1112 1126 else:
1113 1127 print "[Reading] No files"
1114 1128
1115 1129 sys.exit(-1)
1116 1130
1117 1131 # self.updateDataHeader()
1118 1132 if last_set != None:
1119 1133 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1120 1134 return
1121 1135
1122 1136 def getBasicHeader(self):
1123 1137
1124 1138 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1125 1139
1126 1140 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1127 1141
1128 1142 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1129 1143
1130 1144 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1131 1145
1132 1146 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1133 1147
1134 1148 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1135 1149
1136 1150 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1137 1151
1138 1152 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1139 1153
1140 1154
1141 1155 def getFirstHeader(self):
1142 1156
1143 1157 raise ValueError, "This method has not been implemented"
1144 1158
1145 1159 def getData(self):
1146 1160
1147 1161 raise ValueError, "This method has not been implemented"
1148 1162
1149 1163 def hasNotDataInBuffer(self):
1150 1164
1151 1165 raise ValueError, "This method has not been implemented"
1152 1166
1153 1167 def readBlock(self):
1154 1168
1155 1169 raise ValueError, "This method has not been implemented"
1156 1170
1157 1171 def isEndProcess(self):
1158 1172
1159 1173 return self.flagNoMoreFiles
1160 1174
1161 1175 def printReadBlocks(self):
1162 1176
1163 1177 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1164 1178
1165 1179 def printTotalBlocks(self):
1166 1180
1167 1181 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1168 1182
1169 1183 def printNumberOfBlock(self):
1170 1184
1171 1185 if self.flagIsNewBlock:
1172 1186 print "[Reading] Block No. %04d, Total blocks %04d -> %s" %(self.basicHeaderObj.dataBlock, self.nTotalBlocks, self.dataOut.datatime.ctime())
1173 1187 self.dataOut.blocknow = self.basicHeaderObj.dataBlock
1174 1188
1175 1189 def printInfo(self):
1176 1190
1177 1191 if self.__printInfo == False:
1178 1192 return
1179 1193
1180 1194 self.basicHeaderObj.printInfo()
1181 1195 self.systemHeaderObj.printInfo()
1182 1196 self.radarControllerHeaderObj.printInfo()
1183 1197 self.processingHeaderObj.printInfo()
1184 1198
1185 1199 self.__printInfo = False
1186 1200
1187 1201
1188 1202 def run(self, **kwargs):
1189 1203
1190 1204 if not(self.isConfig):
1191 1205
1192 1206 # self.dataOut = dataOut
1193 1207 self.setup(**kwargs)
1194 1208 self.isConfig = True
1195 1209
1196 1210 self.getData()
1197 1211
1198 1212 class JRODataWriter(JRODataIO):
1199 1213
1200 1214 """
1201 1215 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1202 1216 de los datos siempre se realiza por bloques.
1203 1217 """
1204 1218
1205 1219 blockIndex = 0
1206 1220
1207 1221 path = None
1208 1222
1209 1223 setFile = None
1210 1224
1211 1225 profilesPerBlock = None
1212 1226
1213 1227 blocksPerFile = None
1214 1228
1215 1229 nWriteBlocks = 0
1216 1230
1217 1231 def __init__(self, dataOut=None):
1218 1232 raise ValueError, "Not implemented"
1219 1233
1220 1234
1221 1235 def hasAllDataInBuffer(self):
1222 1236 raise ValueError, "Not implemented"
1223 1237
1224 1238
1225 1239 def setBlockDimension(self):
1226 1240 raise ValueError, "Not implemented"
1227 1241
1228 1242
1229 1243 def writeBlock(self):
1230 1244 raise ValueError, "No implemented"
1231 1245
1232 1246
1233 1247 def putData(self):
1234 1248 raise ValueError, "No implemented"
1235 1249
1236 1250
1237 1251 def setBasicHeader(self):
1238 1252
1239 1253 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1240 1254 self.basicHeaderObj.version = self.versionFile
1241 1255 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1242 1256
1243 1257 utc = numpy.floor(self.dataOut.utctime)
1244 1258 milisecond = (self.dataOut.utctime - utc)* 1000.0
1245 1259
1246 1260 self.basicHeaderObj.utc = utc
1247 1261 self.basicHeaderObj.miliSecond = milisecond
1248 1262 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1249 1263 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1250 1264 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1251 1265
1252 1266 def setFirstHeader(self):
1253 1267 """
1254 1268 Obtiene una copia del First Header
1255 1269
1256 1270 Affected:
1257 1271
1258 1272 self.basicHeaderObj
1259 1273 self.systemHeaderObj
1260 1274 self.radarControllerHeaderObj
1261 1275 self.processingHeaderObj self.
1262 1276
1263 1277 Return:
1264 1278 None
1265 1279 """
1266 1280
1267 1281 raise ValueError, "No implemented"
1268 1282
1269 1283 def __writeFirstHeader(self):
1270 1284 """
1271 1285 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1272 1286
1273 1287 Affected:
1274 1288 __dataType
1275 1289
1276 1290 Return:
1277 1291 None
1278 1292 """
1279 1293
1280 1294 # CALCULAR PARAMETROS
1281 1295
1282 1296 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1283 1297 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1284 1298
1285 1299 self.basicHeaderObj.write(self.fp)
1286 1300 self.systemHeaderObj.write(self.fp)
1287 1301 self.radarControllerHeaderObj.write(self.fp)
1288 1302 self.processingHeaderObj.write(self.fp)
1289 1303
1290 1304 self.dtype = self.dataOut.dtype
1291 1305
1292 1306 def __setNewBlock(self):
1293 1307 """
1294 1308 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1295 1309
1296 1310 Return:
1297 1311 0 : si no pudo escribir nada
1298 1312 1 : Si escribio el Basic el First Header
1299 1313 """
1300 1314 if self.fp == None:
1301 1315 self.setNextFile()
1302 1316
1303 1317 if self.flagIsNewFile:
1304 1318 return 1
1305 1319
1306 1320 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1307 1321 self.basicHeaderObj.write(self.fp)
1308 1322 return 1
1309 1323
1310 1324 if not( self.setNextFile() ):
1311 1325 return 0
1312 1326
1313 1327 return 1
1314 1328
1315 1329
1316 1330 def writeNextBlock(self):
1317 1331 """
1318 1332 Selecciona el bloque siguiente de datos y los escribe en un file
1319 1333
1320 1334 Return:
1321 1335 0 : Si no hizo pudo escribir el bloque de datos
1322 1336 1 : Si no pudo escribir el bloque de datos
1323 1337 """
1324 1338 if not( self.__setNewBlock() ):
1325 1339 return 0
1326 1340
1327 1341 self.writeBlock()
1328 1342
1329 1343 return 1
1330 1344
1331 1345 def setNextFile(self):
1332 1346 """
1333 1347 Determina el siguiente file que sera escrito
1334 1348
1335 1349 Affected:
1336 1350 self.filename
1337 1351 self.subfolder
1338 1352 self.fp
1339 1353 self.setFile
1340 1354 self.flagIsNewFile
1341 1355
1342 1356 Return:
1343 1357 0 : Si el archivo no puede ser escrito
1344 1358 1 : Si el archivo esta listo para ser escrito
1345 1359 """
1346 1360 ext = self.ext
1347 1361 path = self.path
1348 1362
1349 1363 if self.fp != None:
1350 1364 self.fp.close()
1351 1365
1352 1366 timeTuple = time.localtime( self.dataOut.utctime)
1353 1367 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1354 1368
1355 1369 fullpath = os.path.join( path, subfolder )
1356 1370 if not( os.path.exists(fullpath) ):
1357 1371 os.mkdir(fullpath)
1358 1372 self.setFile = -1 #inicializo mi contador de seteo
1359 1373 else:
1360 1374 filesList = os.listdir( fullpath )
1361 1375 if len( filesList ) > 0:
1362 1376 filesList = sorted( filesList, key=str.lower )
1363 1377 filen = filesList[-1]
1364 1378 # el filename debera tener el siguiente formato
1365 1379 # 0 1234 567 89A BCDE (hex)
1366 1380 # x YYYY DDD SSS .ext
1367 1381 if isNumber( filen[8:11] ):
1368 1382 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1369 1383 else:
1370 1384 self.setFile = -1
1371 1385 else:
1372 1386 self.setFile = -1 #inicializo mi contador de seteo
1373 1387
1374 1388 setFile = self.setFile
1375 1389 setFile += 1
1376 1390
1377 1391 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
1378 1392 timeTuple.tm_year,
1379 1393 timeTuple.tm_yday,
1380 1394 setFile,
1381 1395 ext )
1382 1396
1383 1397 filename = os.path.join( path, subfolder, filen )
1384 1398
1385 1399 fp = open( filename,'wb' )
1386 1400
1387 1401 self.blockIndex = 0
1388 1402
1389 1403 #guardando atributos
1390 1404 self.filename = filename
1391 1405 self.subfolder = subfolder
1392 1406 self.fp = fp
1393 1407 self.setFile = setFile
1394 1408 self.flagIsNewFile = 1
1395 1409
1396 1410 self.setFirstHeader()
1397 1411
1398 1412 print '[Writing] file: %s'%self.filename
1399 1413
1400 1414 self.__writeFirstHeader()
1401 1415
1402 1416 return 1
1403 1417
1404 1418 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=0, ext=None):
1405 1419 """
1406 1420 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1407 1421
1408 1422 Inputs:
1409 1423 path : el path destino en el cual se escribiran los files a crear
1410 1424 format : formato en el cual sera salvado un file
1411 1425 set : el setebo del file
1412 1426
1413 1427 Return:
1414 1428 0 : Si no realizo un buen seteo
1415 1429 1 : Si realizo un buen seteo
1416 1430 """
1417 1431
1418 1432 if ext == None:
1419 1433 ext = self.ext
1420 1434
1421 1435 ext = ext.lower()
1422 1436
1423 1437 self.ext = ext
1424 1438
1425 1439 self.path = path
1426 1440
1427 1441 self.setFile = set - 1
1428 1442
1429 1443 self.blocksPerFile = blocksPerFile
1430 1444
1431 1445 self.profilesPerBlock = profilesPerBlock
1432 1446
1433 1447 self.dataOut = dataOut
1434 1448
1435 1449 if not(self.setNextFile()):
1436 1450 print "[Writing] There isn't a next file"
1437 1451 return 0
1438 1452
1439 1453 self.setBlockDimension()
1440 1454
1441 1455 return 1
1442 1456
1443 1457 def run(self, dataOut, **kwargs):
1444 1458
1445 1459 if not(self.isConfig):
1446 1460
1447 1461 self.setup(dataOut, **kwargs)
1448 1462 self.isConfig = True
1449 1463
1450 1464 self.putData()
1451 1465
Show file before | Show file after | Hide comments
@@ -1,583 +1,590
1 1 '''
2 2 Created on Jul 3, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 import os
6 7 import datetime
7 8 import numpy
8 9
9 10 try:
10 11 from gevent import sleep
11 12 except:
12 13 from time import sleep
13 14
14 15 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
15 16 from schainpy.model.data.jrodata import Voltage
16 17 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
17 18
18 19 try:
19 20 import digital_rf_hdf5
20 21 except:
21 22 print 'You should install "digital_rf_hdf5" module if you want to read USRP data'
22 23
23 24 class USRPReader(ProcessingUnit):
24 25 '''
25 26 classdocs
26 27 '''
27 28
28 29 def __init__(self):
29 30 '''
30 31 Constructor
31 32 '''
32 33
33 34 ProcessingUnit.__init__(self)
34 35
35 36 self.dataOut = Voltage()
36 37 self.__printInfo = True
37 38 self.__flagDiscontinuousBlock = False
38 39 self.__bufferIndex = 9999999
39 40
40 41 self.__ippKm = None
41 42 self.__codeType = 0
42 43 self.__nCode = None
43 44 self.__nBaud = None
44 45 self.__code = None
45 46
46 47 def __getCurrentSecond(self):
47 48
48 49 return self.__thisUnixSample/self.__sample_rate
49 50
50 51 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
51 52
52 53 def __setFileHeader(self):
53 54 '''
54 55 In this method will be initialized every parameter of dataOut object (header, no data)
55 56 '''
56 57 nProfiles = self.__sample_rate #Number of profiles by second
57 58
58 59 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
59 60 txA=0,
60 61 txB=0,
61 62 nWindows=1,
62 63 nHeights=self.__nSamples,
63 64 firstHeight=self.__firstHeigth,
64 65 deltaHeight=self.__deltaHeigth,
65 66 codeType=self.__codeType,
66 67 nCode=self.__nCode, nBaud=self.__nBaud,
67 68 code = self.__code)
68 69
69 70 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
70 71 nProfiles=nProfiles,
71 72 nChannels=len(self.__channelList),
72 73 adcResolution=14)
73 74
74 75 self.dataOut.type = "Voltage"
75 76
76 77 self.dataOut.data = None
77 78
78 79 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
79 80
80 81 # self.dataOut.nChannels = 0
81 82
82 83 # self.dataOut.nHeights = 0
83 84
84 85 self.dataOut.nProfiles = nProfiles
85 86
86 87 self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
87 88
88 89 self.dataOut.channelList = self.__channelList
89 90
90 91 self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
91 92
92 93 # self.dataOut.channelIndexList = None
93 94
94 95 self.dataOut.flagNoData = True
95 96
96 97 #Set to TRUE if the data is discontinuous
97 98 self.dataOut.flagDiscontinuousBlock = False
98 99
99 100 self.dataOut.utctime = None
100 101
101 102 self.dataOut.timeZone = self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
102 103
103 104 self.dataOut.dstFlag = 0
104 105
105 106 self.dataOut.errorCount = 0
106 107
107 108 self.dataOut.nCohInt = 1
108 109
109 110 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
110 111
111 112 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
112 113
113 114 self.dataOut.flagShiftFFT = False
114 115
115 116 self.dataOut.ippSeconds = 1.0*self.__nSamples/self.__sample_rate
116 117
117 118 #Time interval between profiles
118 119 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
119 120
120 121 self.dataOut.frequency = self.__frequency
121 122
122 123 self.dataOut.realtime = self.__online
123 124
124 125 def findDatafiles(self, path, startDate=None, endDate=None):
125 126
127 if not os.path.isdir(path):
128 return []
129
126 130 try:
127 131 digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
128 132 except:
129 133 digitalReadObj = digital_rf_hdf5.read_hdf5(path)
130 134
131 135 channelNameList = digitalReadObj.get_channels()
132 136
133 137 if not channelNameList:
134 138 return []
135 139
136 140 metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
137 141
138 142 sample_rate = metadata_dict['sample_rate'][0]
139 143
140 144 this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
141 145
142 146 try:
143 147 timezone = this_metadata_file['timezone'].value
144 148 except:
145 149 timezone = 0
146 150
147 151 startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
148 152
149 153 startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
150 154 endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
151 155
152 156 if not startDate:
153 157 startDate = startDatetime.date()
154 158
155 159 if not endDate:
156 160 endDate = endDatatime.date()
157 161
158 162 dateList = []
159 163
160 164 thisDatetime = startDatetime
161 165
162 166 while(thisDatetime<=endDatatime):
163 167
164 168 thisDate = thisDatetime.date()
165 169
166 170 if thisDate < startDate:
167 171 continue
168 172
169 173 if thisDate > endDate:
170 174 break
171 175
172 176 dateList.append(thisDate)
173 177 thisDatetime += datetime.timedelta(1)
174 178
175 179 return dateList
176 180
177 181 def setup(self, path = None,
178 182 startDate = None,
179 183 endDate = None,
180 184 startTime = datetime.time(0,0,0),
181 185 endTime = datetime.time(23,59,59),
182 186 channelList = None,
183 187 nSamples = None,
184 188 ippKm = 60,
185 189 online = False,
186 190 delay = 60,
187 191 buffer_size = None,
188 192 nbuffer = 1024,
189 193 **kwargs):
190 194 '''
191 195 In this method we should set all initial parameters.
192 196
193 197 Inputs:
194 198 path
195 199 startDate
196 200 endDate
197 201 startTime
198 202 endTime
199 203 set
200 204 expLabel
201 205 ext
202 206 online
203 207 delay
204 208 '''
205 209
206 210 if not buffer_size:
207 211 buffer_size = nbuffer
208
212
213 if not os.path.isdir(path):
214 raise ValueError, "[Reading] This path %s does not exist" %path
215
209 216 try:
210 217 self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
211 218 except:
212 219 self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
213 220
214 221 channelNameList = self.digitalReadObj.get_channels()
215 222
216 223 if not channelNameList:
217 224 raise IOError, "[Reading] The path doesn,t have any files .. "
218 225
219 226 if not channelList:
220 227 channelList = range(len(channelNameList))
221 228
222 229 ########## Reading metadata ######################
223 230
224 231 metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
225 232
226 233 self.__sample_rate = metadata_dict['sample_rate'][0]
227 234 self.__samples_per_file = metadata_dict['samples_per_file'][0]
228 235 self.__deltaHeigth = 1e6*0.15/self.__sample_rate
229 236
230 237 this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
231 238
232 239 self.__frequency = this_metadata_file['center_frequencies'].value
233 240 try:
234 241 self.__timezone = this_metadata_file['timezone'].value
235 242 except:
236 243 self.__timezone = 0
237 244
238 245 self.__firstHeigth = 0
239 246
240 247 try:
241 248 codeType = this_metadata_file['codeType'].value
242 249 except:
243 250 codeType = 0
244 251
245 252 nCode = 0
246 253 nBaud = 0
247 254 code = None
248 255
249 256 if codeType:
250 257 nCode = this_metadata_file['nCode'].value
251 258 nBaud = this_metadata_file['nBaud'].value
252 259 code = this_metadata_file['code'].value
253 260
254 261 if not ippKm:
255 262 try:
256 263 #seconds to km
257 264 ippKm = 1e6*0.15*this_metadata_file['ipp'].value
258 265 except:
259 266 ippKm = None
260 267
261 268 ####################################################
262 269 startUTCSecond = None
263 270 endUTCSecond = None
264 271
265 272 if startDate:
266 273 startDatetime = datetime.datetime.combine(startDate, startTime)
267 274 startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
268 275
269 276 if endDate:
270 277 endDatetime = datetime.datetime.combine(endDate, endTime)
271 278 endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
272 279
273 280 start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
274 281
275 282 if not startUTCSecond:
276 283 startUTCSecond = start_index/self.__sample_rate
277 284
278 285 if start_index > startUTCSecond*self.__sample_rate:
279 286 startUTCSecond = start_index/self.__sample_rate
280 287
281 288 if not endUTCSecond:
282 289 endUTCSecond = end_index/self.__sample_rate
283 290
284 291 if end_index < endUTCSecond*self.__sample_rate:
285 292 endUTCSecond = end_index/self.__sample_rate
286 293
287 294 if not nSamples:
288 295 if not ippKm:
289 296 raise ValueError, "[Reading] nSamples or ippKm should be defined"
290 297
291 298 nSamples = ippKm / (1e6*0.15/self.__sample_rate)
292 299
293 300 channelBoundList = []
294 301 channelNameListFiltered = []
295 302
296 303 for thisIndexChannel in channelList:
297 304 thisChannelName = channelNameList[thisIndexChannel]
298 305 start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
299 306 channelBoundList.append((start_index, end_index))
300 307 channelNameListFiltered.append(thisChannelName)
301 308
302 309 self.profileIndex = 0
303 310
304 311 self.__delay = delay
305 312 self.__ippKm = ippKm
306 313 self.__codeType = codeType
307 314 self.__nCode = nCode
308 315 self.__nBaud = nBaud
309 316 self.__code = code
310 317
311 318 self.__datapath = path
312 319 self.__online = online
313 320 self.__channelList = channelList
314 321 self.__channelNameList = channelNameListFiltered
315 322 self.__channelBoundList = channelBoundList
316 323 self.__nSamples = nSamples
317 324 self.__samples_to_read = buffer_size*nSamples
318 325 self.__nChannels = len(self.__channelList)
319 326
320 327 self.__startUTCSecond = startUTCSecond
321 328 self.__endUTCSecond = endUTCSecond
322 329
323 330 self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
324 331
325 332 if online:
326 333 # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
327 334 startUTCSecond = numpy.floor(endUTCSecond)
328 335
329 336 self.__thisUnixSample = int(startUTCSecond*self.__sample_rate) - self.__samples_to_read
330 337
331 338 self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
332 339
333 340 self.__setFileHeader()
334 341 self.isConfig = True
335 342
336 343 print "[Reading] USRP Data was found from %s to %s " %(
337 344 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
338 345 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
339 346 )
340 347
341 348 print "[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
342 349 datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
343 350 )
344 351
345 352 def __reload(self):
346 353
347 354 if not self.__online:
348 355 return
349 356
350 357 # print
351 358 # print "%s not in range [%s, %s]" %(
352 359 # datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
353 360 # datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
354 361 # datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
355 362 # )
356 363 print "[Reading] reloading metadata ..."
357 364
358 365 try:
359 366 self.digitalReadObj.reload(complete_update=True)
360 367 except:
361 368 self.digitalReadObj.reload()
362 369
363 370 start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
364 371
365 372 if start_index > self.__startUTCSecond*self.__sample_rate:
366 373 self.__startUTCSecond = 1.0*start_index/self.__sample_rate
367 374
368 375 if end_index > self.__endUTCSecond*self.__sample_rate:
369 376 self.__endUTCSecond = 1.0*end_index/self.__sample_rate
370 377 print
371 378 print "[Reading] New timerange found [%s, %s] " %(
372 379 datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
373 380 datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
374 381 )
375 382
376 383 return True
377 384
378 385 return False
379 386
380 387 def __readNextBlock(self, seconds=30, volt_scale = 218776):
381 388 '''
382 389 '''
383 390
384 391 #Set the next data
385 392 self.__flagDiscontinuousBlock = False
386 393 self.__thisUnixSample += self.__samples_to_read
387 394
388 395 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
389 396 print "[Reading] There are no more data into selected timerange"
390 397
391 398 self.__reload()
392 399
393 400 if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
394 401 self.__thisUnixSample -= self.__samples_to_read
395 402 return False
396 403
397 404 indexChannel = 0
398 405
399 406 dataOk = False
400 407
401 408 for thisChannelName in self.__channelNameList:
402 409
403 410 try:
404 411 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
405 412 self.__samples_to_read,
406 413 thisChannelName)
407 414
408 415 except IOError, e:
409 416 #read next profile
410 417 self.__flagDiscontinuousBlock = True
411 418 print e
412 419 break
413 420
414 421 if result.shape[0] != self.__samples_to_read:
415 422 self.__flagDiscontinuousBlock = True
416 423 print "[Reading] %s: Too few samples were found, just %d samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
417 424 result.shape[0])
418 425 break
419 426
420 427 self.__data_buffer[indexChannel,:] = result*volt_scale
421 428
422 429 indexChannel += 1
423 430
424 431 dataOk = True
425 432
426 433 self.__utctime = self.__thisUnixSample/self.__sample_rate
427 434
428 435 if not dataOk:
429 436 return False
430 437
431 438 print "[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
432 439 self.__samples_to_read,
433 440 self.__timeInterval)
434 441
435 442 self.__bufferIndex = 0
436 443
437 444 return True
438 445
439 446 def __isBufferEmpty(self):
440 447
441 448 if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
442 449 return False
443 450
444 451 return True
445 452
446 453 def getData(self, seconds=30, nTries=5):
447 454
448 455 '''
449 456 This method gets the data from files and put the data into the dataOut object
450 457
451 458 In addition, increase el the buffer counter in one.
452 459
453 460 Return:
454 461 data : retorna un perfil de voltages (alturas * canales) copiados desde el
455 462 buffer. Si no hay mas archivos a leer retorna None.
456 463
457 464 Affected:
458 465 self.dataOut
459 466 self.profileIndex
460 467 self.flagDiscontinuousBlock
461 468 self.flagIsNewBlock
462 469 '''
463 470
464 471 err_counter = 0
465 472 self.dataOut.flagNoData = True
466 473
467 474 if self.__isBufferEmpty():
468 475
469 476 self.__flagDiscontinuousBlock = False
470 477
471 478 while True:
472 479 if self.__readNextBlock():
473 480 break
474 481
475 482 if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
476 483 return False
477 484
478 485 if self.__flagDiscontinuousBlock:
479 486 print '[Reading] discontinuous block found ... continue with the next block'
480 487 continue
481 488
482 489 if not self.__online:
483 490 return False
484 491
485 492 err_counter += 1
486 493 if err_counter > nTries:
487 494 return False
488 495
489 496 print '[Reading] waiting %d seconds to read a new block' %seconds
490 497 sleep(seconds)
491 498
492 499 self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
493 500 self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
494 501 self.dataOut.flagNoData = False
495 502 self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
496 503
497 504 self.__bufferIndex += self.__nSamples
498 505 self.profileIndex += 1
499 506
500 507 return True
501 508
502 509 def printInfo(self):
503 510 '''
504 511 '''
505 512 if self.__printInfo == False:
506 513 return
507 514
508 515 # self.systemHeaderObj.printInfo()
509 516 # self.radarControllerHeaderObj.printInfo()
510 517
511 518 self.__printInfo = False
512 519
513 520 def printNumberOfBlock(self):
514 521 '''
515 522 '''
516 523
517 524 print self.profileIndex
518 525
519 526 def run(self, **kwargs):
520 527 '''
521 528 This method will be called many times so here you should put all your code
522 529 '''
523 530
524 531 if not self.isConfig:
525 532 self.setup(**kwargs)
526 533
527 534 self.getData(seconds=self.__delay)
528 535
529 536 return
530 537
531 538 class USRPWriter(Operation):
532 539 '''
533 540 classdocs
534 541 '''
535 542
536 543 def __init__(self):
537 544 '''
538 545 Constructor
539 546 '''
540 547 self.dataOut = None
541 548
542 549 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
543 550 '''
544 551 In this method we should set all initial parameters.
545 552
546 553 Input:
547 554 dataIn : Input data will also be outputa data
548 555
549 556 '''
550 557 self.dataOut = dataIn
551 558
552 559
553 560
554 561
555 562
556 563 self.isConfig = True
557 564
558 565 return
559 566
560 567 def run(self, dataIn, **kwargs):
561 568 '''
562 569 This method will be called many times so here you should put all your code
563 570
564 571 Inputs:
565 572
566 573 dataIn : object with the data
567 574
568 575 '''
569 576
570 577 if not self.isConfig:
571 578 self.setup(dataIn, **kwargs)
572 579
573 580
574 581 if __name__ == '__main__':
575 582
576 583 readObj = USRPReader()
577 584
578 585 while True:
579 586 readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
580 587 # readObj.printInfo()
581 588 readObj.printNumberOfBlock()
582 589
583 590 No newline at end of file
General Comments 0
You need to be logged in to leave comments. Login now