##// END OF EJS Templates
schain-jc
RSS
falta
José Chávez -
r1059:5e064f9e5ed9
parent child
Show More
Side by Side Unified
Context file:
r1059:5e064f9e5ed9 Loading diff...:
Show file before | Show file after | Hide comments
@@ -1,1280 +1,1280
1 1 '''
2 2 Created on September , 2012
3 3 @author:
4 4 '''
5 5
6 6 import sys
7 7 import ast
8 8 import datetime
9 9 import traceback
10 10 import schainpy
11 11 import schainpy.admin
12 12 from schainpy.utils.log import logToFile
13 13
14 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
15 15 from xml.dom import minidom
16 16
17 from multiprocessing import cpu_count
17 18 from schainpy.model import *
18 19 from time import sleep
19 20
20 21
21
22 22 def prettify(elem):
23 23 """Return a pretty-printed XML string for the Element.
24 24 """
25 25 rough_string = tostring(elem, 'utf-8')
26 26 reparsed = minidom.parseString(rough_string)
27 27 return reparsed.toprettyxml(indent=" ")
28 28
29 29 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
30 30 skip = 0
31 31 cursor = 0
32 32 nFiles = None
33 33 processes = []
34 34 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
35 35 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
36 36 days = (dt2 - dt1).days
37 37
38 38 for day in range(days+1):
39 39 skip = 0
40 40 cursor = 0
41 41 q = Queue()
42 42 processes = []
43 43 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
44 44 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
45 45 firstProcess.start()
46 46 if by_day:
47 47 continue
48 48 nFiles = q.get()
49 49 if nFiles==0:
50 50 continue
51 51 firstProcess.terminate()
52 52 skip = int(math.ceil(nFiles/nProcess))
53 53 while True:
54 54 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
55 55 processes[cursor].start()
56 56 if nFiles < cursor*skip:
57 57 break
58 58 cursor += 1
59 59
60 60 def beforeExit(exctype, value, trace):
61 61 for process in processes:
62 62 process.terminate()
63 63 process.join()
64 64 print traceback.print_tb(trace)
65 65
66 66 sys.excepthook = beforeExit
67 67
68 68 for process in processes:
69 69 process.join()
70 70 process.terminate()
71 71
72 72 time.sleep(3)
73 73
74 74
75 75 class ParameterConf():
76 76
77 77 id = None
78 78 name = None
79 79 value = None
80 80 format = None
81 81
82 82 __formated_value = None
83 83
84 84 ELEMENTNAME = 'Parameter'
85 85
86 86 def __init__(self):
87 87
88 88 self.format = 'str'
89 89
90 90 def getElementName(self):
91 91
92 92 return self.ELEMENTNAME
93 93
94 94 def getValue(self):
95 95
96 96 value = self.value
97 97 format = self.format
98 98
99 99 if self.__formated_value != None:
100 100
101 101 return self.__formated_value
102 102
103 103 if format == 'str':
104 104 self.__formated_value = str(value)
105 105 return self.__formated_value
106 106
107 107 if value == '':
108 108 raise ValueError, "%s: This parameter value is empty" %self.name
109 109
110 110 if format == 'list':
111 111 strList = value.split(',')
112 112
113 113 self.__formated_value = strList
114 114
115 115 return self.__formated_value
116 116
117 117 if format == 'intlist':
118 118 """
119 119 Example:
120 120 value = (0,1,2)
121 121 """
122 122
123 123 new_value = ast.literal_eval(value)
124 124
125 125 if type(new_value) not in (tuple, list):
126 126 new_value = [int(new_value)]
127 127
128 128 self.__formated_value = new_value
129 129
130 130 return self.__formated_value
131 131
132 132 if format == 'floatlist':
133 133 """
134 134 Example:
135 135 value = (0.5, 1.4, 2.7)
136 136 """
137 137
138 138 new_value = ast.literal_eval(value)
139 139
140 140 if type(new_value) not in (tuple, list):
141 141 new_value = [float(new_value)]
142 142
143 143 self.__formated_value = new_value
144 144
145 145 return self.__formated_value
146 146
147 147 if format == 'date':
148 148 strList = value.split('/')
149 149 intList = [int(x) for x in strList]
150 150 date = datetime.date(intList[0], intList[1], intList[2])
151 151
152 152 self.__formated_value = date
153 153
154 154 return self.__formated_value
155 155
156 156 if format == 'time':
157 157 strList = value.split(':')
158 158 intList = [int(x) for x in strList]
159 159 time = datetime.time(intList[0], intList[1], intList[2])
160 160
161 161 self.__formated_value = time
162 162
163 163 return self.__formated_value
164 164
165 165 if format == 'pairslist':
166 166 """
167 167 Example:
168 168 value = (0,1),(1,2)
169 169 """
170 170
171 171 new_value = ast.literal_eval(value)
172 172
173 173 if type(new_value) not in (tuple, list):
174 174 raise ValueError, "%s has to be a tuple or list of pairs" %value
175 175
176 176 if type(new_value[0]) not in (tuple, list):
177 177 if len(new_value) != 2:
178 178 raise ValueError, "%s has to be a tuple or list of pairs" %value
179 179 new_value = [new_value]
180 180
181 181 for thisPair in new_value:
182 182 if len(thisPair) != 2:
183 183 raise ValueError, "%s has to be a tuple or list of pairs" %value
184 184
185 185 self.__formated_value = new_value
186 186
187 187 return self.__formated_value
188 188
189 189 if format == 'multilist':
190 190 """
191 191 Example:
192 192 value = (0,1,2),(3,4,5)
193 193 """
194 194 multiList = ast.literal_eval(value)
195 195
196 196 if type(multiList[0]) == int:
197 197 multiList = ast.literal_eval("(" + value + ")")
198 198
199 199 self.__formated_value = multiList
200 200
201 201 return self.__formated_value
202 202
203 203 if format == 'bool':
204 204 value = int(value)
205 205
206 206 if format == 'int':
207 207 value = float(value)
208 208
209 209 format_func = eval(format)
210 210
211 211 self.__formated_value = format_func(value)
212 212
213 213 return self.__formated_value
214 214
215 215 def updateId(self, new_id):
216 216
217 217 self.id = str(new_id)
218 218
219 219 def setup(self, id, name, value, format='str'):
220 220
221 221 self.id = str(id)
222 222 self.name = name
223 223 self.value = str(value)
224 224 self.format = str.lower(format)
225 225
226 226 self.getValue()
227 227
228 228 return 1
229 229
230 230 def update(self, name, value, format='str'):
231 231
232 232 self.name = name
233 233 self.value = str(value)
234 234 self.format = format
235 235
236 236 def makeXml(self, opElement):
237 237
238 238 parmElement = SubElement(opElement, self.ELEMENTNAME)
239 239 parmElement.set('id', str(self.id))
240 240 parmElement.set('name', self.name)
241 241 parmElement.set('value', self.value)
242 242 parmElement.set('format', self.format)
243 243
244 244 def readXml(self, parmElement):
245 245
246 246 self.id = parmElement.get('id')
247 247 self.name = parmElement.get('name')
248 248 self.value = parmElement.get('value')
249 249 self.format = str.lower(parmElement.get('format'))
250 250
251 251 #Compatible with old signal chain version
252 252 if self.format == 'int' and self.name == 'idfigure':
253 253 self.name = 'id'
254 254
255 255 def printattr(self):
256 256
257 257 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
258 258
259 259 class OperationConf():
260 260
261 261 id = None
262 262 name = None
263 263 priority = None
264 264 type = None
265 265
266 266 parmConfObjList = []
267 267
268 268 ELEMENTNAME = 'Operation'
269 269
270 270 def __init__(self):
271 271
272 272 self.id = '0'
273 273 self.name = None
274 274 self.priority = None
275 275 self.type = 'self'
276 276
277 277
278 278 def __getNewId(self):
279 279
280 280 return int(self.id)*10 + len(self.parmConfObjList) + 1
281 281
282 282 def updateId(self, new_id):
283 283
284 284 self.id = str(new_id)
285 285
286 286 n = 1
287 287 for parmObj in self.parmConfObjList:
288 288
289 289 idParm = str(int(new_id)*10 + n)
290 290 parmObj.updateId(idParm)
291 291
292 292 n += 1
293 293
294 294 def getElementName(self):
295 295
296 296 return self.ELEMENTNAME
297 297
298 298 def getParameterObjList(self):
299 299
300 300 return self.parmConfObjList
301 301
302 302 def getParameterObj(self, parameterName):
303 303
304 304 for parmConfObj in self.parmConfObjList:
305 305
306 306 if parmConfObj.name != parameterName:
307 307 continue
308 308
309 309 return parmConfObj
310 310
311 311 return None
312 312
313 313 def getParameterObjfromValue(self, parameterValue):
314 314
315 315 for parmConfObj in self.parmConfObjList:
316 316
317 317 if parmConfObj.getValue() != parameterValue:
318 318 continue
319 319
320 320 return parmConfObj.getValue()
321 321
322 322 return None
323 323
324 324 def getParameterValue(self, parameterName):
325 325
326 326 parameterObj = self.getParameterObj(parameterName)
327 327
328 328 # if not parameterObj:
329 329 # return None
330 330
331 331 value = parameterObj.getValue()
332 332
333 333 return value
334 334
335 335 def setup(self, id, name, priority, type):
336 336
337 337 self.id = str(id)
338 338 self.name = name
339 339 self.type = type
340 340 self.priority = priority
341 341
342 342 self.parmConfObjList = []
343 343
344 344 def removeParameters(self):
345 345
346 346 for obj in self.parmConfObjList:
347 347 del obj
348 348
349 349 self.parmConfObjList = []
350 350
351 351 def addParameter(self, name, value, format='str'):
352 352
353 353 id = self.__getNewId()
354 354
355 355 parmConfObj = ParameterConf()
356 356 if not parmConfObj.setup(id, name, value, format):
357 357 return None
358 358
359 359 self.parmConfObjList.append(parmConfObj)
360 360
361 361 return parmConfObj
362 362
363 363 def changeParameter(self, name, value, format='str'):
364 364
365 365 parmConfObj = self.getParameterObj(name)
366 366 parmConfObj.update(name, value, format)
367 367
368 368 return parmConfObj
369 369
370 370 def makeXml(self, procUnitElement):
371 371
372 372 opElement = SubElement(procUnitElement, self.ELEMENTNAME)
373 373 opElement.set('id', str(self.id))
374 374 opElement.set('name', self.name)
375 375 opElement.set('type', self.type)
376 376 opElement.set('priority', str(self.priority))
377 377
378 378 for parmConfObj in self.parmConfObjList:
379 379 parmConfObj.makeXml(opElement)
380 380
381 381 def readXml(self, opElement):
382 382
383 383 self.id = opElement.get('id')
384 384 self.name = opElement.get('name')
385 385 self.type = opElement.get('type')
386 386 self.priority = opElement.get('priority')
387 387
388 388 #Compatible with old signal chain version
389 389 #Use of 'run' method instead 'init'
390 390 if self.type == 'self' and self.name == 'init':
391 391 self.name = 'run'
392 392
393 393 self.parmConfObjList = []
394 394
395 395 parmElementList = opElement.iter(ParameterConf().getElementName())
396 396
397 397 for parmElement in parmElementList:
398 398 parmConfObj = ParameterConf()
399 399 parmConfObj.readXml(parmElement)
400 400
401 401 #Compatible with old signal chain version
402 402 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
403 403 if self.type != 'self' and self.name == 'Plot':
404 404 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
405 405 self.name = parmConfObj.value
406 406 continue
407 407
408 408 self.parmConfObjList.append(parmConfObj)
409 409
410 410 def printattr(self):
411 411
412 412 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
413 413 self.id,
414 414 self.name,
415 415 self.type,
416 416 self.priority)
417 417
418 418 for parmConfObj in self.parmConfObjList:
419 419 parmConfObj.printattr()
420 420
421 421 def createObject(self, plotter_queue=None):
422 422
423 423 if self.type == 'self':
424 424 raise ValueError, "This operation type cannot be created"
425 425
426 426 if self.type == 'plotter':
427 427 #Plotter(plotter_name)
428 428 if not plotter_queue:
429 429 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
430 430
431 431 opObj = Plotter(self.name, plotter_queue)
432 432
433 433 if self.type == 'external' or self.type == 'other':
434 434 className = eval(self.name)
435 435 opObj = className()
436 436
437 437 return opObj
438 438
439 439 class ProcUnitConf():
440 440
441 441 id = None
442 442 name = None
443 443 datatype = None
444 444 inputId = None
445 445 parentId = None
446 446
447 447 opConfObjList = []
448 448
449 449 procUnitObj = None
450 450 opObjList = []
451 451
452 452 ELEMENTNAME = 'ProcUnit'
453 453
454 454 def __init__(self):
455 455
456 456 self.id = None
457 457 self.datatype = None
458 458 self.name = None
459 459 self.inputId = None
460 460
461 461 self.opConfObjList = []
462 462
463 463 self.procUnitObj = None
464 464 self.opObjDict = {}
465 465
466 466 def __getPriority(self):
467 467
468 468 return len(self.opConfObjList)+1
469 469
470 470 def __getNewId(self):
471 471
472 472 return int(self.id)*10 + len(self.opConfObjList) + 1
473 473
474 474 def getElementName(self):
475 475
476 476 return self.ELEMENTNAME
477 477
478 478 def getId(self):
479 479
480 480 return self.id
481 481
482 482 def updateId(self, new_id, parentId=parentId):
483 483
484 484
485 485 new_id = int(parentId)*10 + (int(self.id) % 10)
486 486 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
487 487
488 488 #If this proc unit has not inputs
489 489 if self.inputId == '0':
490 490 new_inputId = 0
491 491
492 492 n = 1
493 493 for opConfObj in self.opConfObjList:
494 494
495 495 idOp = str(int(new_id)*10 + n)
496 496 opConfObj.updateId(idOp)
497 497
498 498 n += 1
499 499
500 500 self.parentId = str(parentId)
501 501 self.id = str(new_id)
502 502 self.inputId = str(new_inputId)
503 503
504 504
505 505 def getInputId(self):
506 506
507 507 return self.inputId
508 508
509 509 def getOperationObjList(self):
510 510
511 511 return self.opConfObjList
512 512
513 513 def getOperationObj(self, name=None):
514 514
515 515 for opConfObj in self.opConfObjList:
516 516
517 517 if opConfObj.name != name:
518 518 continue
519 519
520 520 return opConfObj
521 521
522 522 return None
523 523
524 524 def getOpObjfromParamValue(self, value=None):
525 525
526 526 for opConfObj in self.opConfObjList:
527 527 if opConfObj.getParameterObjfromValue(parameterValue=value) != value:
528 528 continue
529 529 return opConfObj
530 530 return None
531 531
532 532 def getProcUnitObj(self):
533 533
534 534 return self.procUnitObj
535 535
536 536 def setup(self, id, name, datatype, inputId, parentId=None):
537 537
538 538 #Compatible with old signal chain version
539 539 if datatype==None and name==None:
540 540 raise ValueError, "datatype or name should be defined"
541 541
542 542 if name==None:
543 543 if 'Proc' in datatype:
544 544 name = datatype
545 545 else:
546 546 name = '%sProc' %(datatype)
547 547
548 548 if datatype==None:
549 549 datatype = name.replace('Proc','')
550 550
551 551 self.id = str(id)
552 552 self.name = name
553 553 self.datatype = datatype
554 554 self.inputId = inputId
555 555 self.parentId = parentId
556 556
557 557 self.opConfObjList = []
558 558
559 559 self.addOperation(name='run', optype='self')
560 560
561 561 def removeOperations(self):
562 562
563 563 for obj in self.opConfObjList:
564 564 del obj
565 565
566 566 self.opConfObjList = []
567 567 self.addOperation(name='run')
568 568
569 569 def addParameter(self, **kwargs):
570 570 '''
571 571 Add parameters to "run" operation
572 572 '''
573 573 opObj = self.opConfObjList[0]
574 574
575 575 opObj.addParameter(**kwargs)
576 576
577 577 return opObj
578 578
579 579 def addOperation(self, name, optype='self'):
580 580
581 581 id = self.__getNewId()
582 582 priority = self.__getPriority()
583 583
584 584 opConfObj = OperationConf()
585 585 opConfObj.setup(id, name=name, priority=priority, type=optype)
586 586
587 587 self.opConfObjList.append(opConfObj)
588 588
589 589 return opConfObj
590 590
591 591 def makeXml(self, projectElement):
592 592
593 593 procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
594 594 procUnitElement.set('id', str(self.id))
595 595 procUnitElement.set('name', self.name)
596 596 procUnitElement.set('datatype', self.datatype)
597 597 procUnitElement.set('inputId', str(self.inputId))
598 598
599 599 for opConfObj in self.opConfObjList:
600 600 opConfObj.makeXml(procUnitElement)
601 601
602 602 def readXml(self, upElement):
603 603
604 604 self.id = upElement.get('id')
605 605 self.name = upElement.get('name')
606 606 self.datatype = upElement.get('datatype')
607 607 self.inputId = upElement.get('inputId')
608 608
609 609 if self.ELEMENTNAME == "ReadUnit":
610 610 self.datatype = self.datatype.replace("Reader", "")
611 611
612 612 if self.ELEMENTNAME == "ProcUnit":
613 613 self.datatype = self.datatype.replace("Proc", "")
614 614
615 615 if self.inputId == 'None':
616 616 self.inputId = '0'
617 617
618 618 self.opConfObjList = []
619 619
620 620 opElementList = upElement.iter(OperationConf().getElementName())
621 621
622 622 for opElement in opElementList:
623 623 opConfObj = OperationConf()
624 624 opConfObj.readXml(opElement)
625 625 self.opConfObjList.append(opConfObj)
626 626
627 627 def printattr(self):
628 628
629 629 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
630 630 self.id,
631 631 self.name,
632 632 self.datatype,
633 633 self.inputId)
634 634
635 635 for opConfObj in self.opConfObjList:
636 636 opConfObj.printattr()
637 637
638 638 def createObjects(self, plotter_queue=None):
639 639
640 640 className = eval(self.name)
641 641 procUnitObj = className()
642 642
643 643 for opConfObj in self.opConfObjList:
644 644
645 645 if opConfObj.type == 'self':
646 646 continue
647 647
648 648 opObj = opConfObj.createObject(plotter_queue)
649 649
650 650 self.opObjDict[opConfObj.id] = opObj
651 651 procUnitObj.addOperation(opObj, opConfObj.id)
652 652
653 653 self.procUnitObj = procUnitObj
654 654
655 655 return procUnitObj
656 656
657 657 def run(self):
658 658
659 659 is_ok = False
660 660
661 661 for opConfObj in self.opConfObjList:
662 662
663 663 kwargs = {}
664 664 for parmConfObj in opConfObj.getParameterObjList():
665 665 if opConfObj.name == 'run' and parmConfObj.name == 'datatype':
666 666 continue
667 667
668 668 kwargs[parmConfObj.name] = parmConfObj.getValue()
669 669
670 670 #ini = time.time()
671 671
672 672 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
673 673 sts = self.procUnitObj.call(opType = opConfObj.type,
674 674 opName = opConfObj.name,
675 675 opId = opConfObj.id)
676 676
677 677 # total_time = time.time() - ini
678 678 #
679 679 # if total_time > 0.002:
680 680 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
681 681
682 682 is_ok = is_ok or sts
683 683
684 684 return is_ok
685 685
686 686 def close(self):
687 687
688 688 for opConfObj in self.opConfObjList:
689 689 if opConfObj.type == 'self':
690 690 continue
691 691
692 692 opObj = self.procUnitObj.getOperationObj(opConfObj.id)
693 693 opObj.close()
694 694
695 695 self.procUnitObj.close()
696 696
697 697 return
698 698
699 699 class ReadUnitConf(ProcUnitConf):
700 700
701 701 path = None
702 702 startDate = None
703 703 endDate = None
704 704 startTime = None
705 705 endTime = None
706 706
707 707 ELEMENTNAME = 'ReadUnit'
708 708
709 709 def __init__(self):
710 710
711 711 self.id = None
712 712 self.datatype = None
713 713 self.name = None
714 714 self.inputId = None
715 715
716 716 self.parentId = None
717 717
718 718 self.opConfObjList = []
719 719 self.opObjList = []
720 720
721 721 def getElementName(self):
722 722
723 723 return self.ELEMENTNAME
724 724
725 725 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
726 726 endTime="", parentId=None, queue=None, server=None, **kwargs):
727 727 #Compatible with old signal chain version
728 728 if datatype==None and name==None:
729 729 raise ValueError, "datatype or name should be defined"
730 730
731 731 if name==None:
732 732 if 'Reader' in datatype:
733 733 name = datatype
734 734 else:
735 735 name = '%sReader' %(datatype)
736 736
737 737 if datatype==None:
738 738 datatype = name.replace('Reader','')
739 739
740 740 self.id = id
741 741 self.name = name
742 742 self.datatype = datatype
743 743
744 744 self.path = os.path.abspath(path)
745 745 self.startDate = startDate
746 746 self.endDate = endDate
747 747 self.startTime = startTime
748 748 self.endTime = endTime
749 749
750 750 self.inputId = '0'
751 751 self.parentId = parentId
752 752
753 753 self.addRunOperation(**kwargs)
754 754
755 755 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
756 756
757 757 #Compatible with old signal chain version
758 758 if datatype==None and name==None:
759 759 raise ValueError, "datatype or name should be defined"
760 760
761 761 if name==None:
762 762 if 'Reader' in datatype:
763 763 name = datatype
764 764 else:
765 765 name = '%sReader' %(datatype)
766 766
767 767 if datatype==None:
768 768 datatype = name.replace('Reader','')
769 769
770 770 self.datatype = datatype
771 771 self.name = name
772 772 self.path = path
773 773 self.startDate = startDate
774 774 self.endDate = endDate
775 775 self.startTime = startTime
776 776 self.endTime = endTime
777 777
778 778 self.inputId = '0'
779 779 self.parentId = parentId
780 780
781 781 self.updateRunOperation(**kwargs)
782 782
783 783 def removeOperations(self):
784 784
785 785 for obj in self.opConfObjList:
786 786 del obj
787 787
788 788 self.opConfObjList = []
789 789
790 790 def addRunOperation(self, **kwargs):
791 791
792 792 opObj = self.addOperation(name = 'run', optype = 'self')
793 793
794 794 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
795 795 opObj.addParameter(name='path' , value=self.path, format='str')
796 796 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
797 797 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
798 798 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
799 799 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
800 800
801 801 for key, value in kwargs.items():
802 802 opObj.addParameter(name=key, value=value, format=type(value).__name__)
803 803
804 804 return opObj
805 805
806 806 def updateRunOperation(self, **kwargs):
807 807
808 808 opObj = self.getOperationObj(name = 'run')
809 809 opObj.removeParameters()
810 810
811 811 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
812 812 opObj.addParameter(name='path' , value=self.path, format='str')
813 813 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
814 814 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
815 815 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
816 816 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
817 817
818 818 for key, value in kwargs.items():
819 819 opObj.addParameter(name=key, value=value, format=type(value).__name__)
820 820
821 821 return opObj
822 822
823 823 # def makeXml(self, projectElement):
824 824 #
825 825 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
826 826 # procUnitElement.set('id', str(self.id))
827 827 # procUnitElement.set('name', self.name)
828 828 # procUnitElement.set('datatype', self.datatype)
829 829 # procUnitElement.set('inputId', str(self.inputId))
830 830 #
831 831 # for opConfObj in self.opConfObjList:
832 832 # opConfObj.makeXml(procUnitElement)
833 833
834 834 def readXml(self, upElement):
835 835
836 836 self.id = upElement.get('id')
837 837 self.name = upElement.get('name')
838 838 self.datatype = upElement.get('datatype')
839 839 self.inputId = upElement.get('inputId')
840 840
841 841 if self.ELEMENTNAME == "ReadUnit":
842 842 self.datatype = self.datatype.replace("Reader", "")
843 843
844 844 if self.inputId == 'None':
845 845 self.inputId = '0'
846 846
847 847 self.opConfObjList = []
848 848
849 849 opElementList = upElement.iter(OperationConf().getElementName())
850 850
851 851 for opElement in opElementList:
852 852 opConfObj = OperationConf()
853 853 opConfObj.readXml(opElement)
854 854 self.opConfObjList.append(opConfObj)
855 855
856 856 if opConfObj.name == 'run':
857 857 self.path = opConfObj.getParameterValue('path')
858 858 self.startDate = opConfObj.getParameterValue('startDate')
859 859 self.endDate = opConfObj.getParameterValue('endDate')
860 860 self.startTime = opConfObj.getParameterValue('startTime')
861 861 self.endTime = opConfObj.getParameterValue('endTime')
862 862
863 863 class Project(Process):
864 864 id = None
865 865 name = None
866 866 description = None
867 867 filename = None
868 868
869 869 procUnitConfObjDict = None
870 870
871 871 ELEMENTNAME = 'Project'
872 872
873 873 plotterQueue = None
874 874
875 875 def __init__(self, plotter_queue=None, logfile=None):
876 876 Process.__init__(self)
877 877 self.id = None
878 878 self.name = None
879 879 self.description = None
880 880 if logfile is not None:
881 881 logToFile(logfile)
882 882 self.plotterQueue = plotter_queue
883 883
884 884 self.procUnitConfObjDict = {}
885 885
886 886 def __getNewId(self):
887 887
888 888 idList = self.procUnitConfObjDict.keys()
889 889
890 890 id = int(self.id)*10
891 891
892 892 while True:
893 893 id += 1
894 894
895 895 if str(id) in idList:
896 896 continue
897 897
898 898 break
899 899
900 900 return str(id)
901 901
902 902 def getElementName(self):
903 903
904 904 return self.ELEMENTNAME
905 905
906 906 def getId(self):
907 907
908 908 return self.id
909 909
910 910 def updateId(self, new_id):
911 911
912 912 self.id = str(new_id)
913 913
914 914 keyList = self.procUnitConfObjDict.keys()
915 915 keyList.sort()
916 916
917 917 n = 1
918 918 newProcUnitConfObjDict = {}
919 919
920 920 for procKey in keyList:
921 921
922 922 procUnitConfObj = self.procUnitConfObjDict[procKey]
923 923 idProcUnit = str(int(self.id)*10 + n)
924 924 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
925 925
926 926 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
927 927 n += 1
928 928
929 929 self.procUnitConfObjDict = newProcUnitConfObjDict
930 930
931 931 def setup(self, id, name, description):
932 932
933 933 self.id = str(id)
934 934 self.name = name
935 935 self.description = description
936 936
937 937 def update(self, name, description):
938 938
939 939 self.name = name
940 940 self.description = description
941 941
942 942 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
943 943 if id is None:
944 944 idReadUnit = self.__getNewId()
945 945 else:
946 946 idReadUnit = str(id)
947 947
948 948 readUnitConfObj = ReadUnitConf()
949 949 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
950 950
951 951 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
952 952
953 953 return readUnitConfObj
954 954
955 955 def addProcUnit(self, inputId='0', datatype=None, name=None):
956 956
957 957 idProcUnit = self.__getNewId()
958 958
959 959 procUnitConfObj = ProcUnitConf()
960 960 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
961 961
962 962 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
963 963
964 964 return procUnitConfObj
965 965
966 966 def removeProcUnit(self, id):
967 967
968 968 if id in self.procUnitConfObjDict.keys():
969 969 self.procUnitConfObjDict.pop(id)
970 970
971 971 def getReadUnitId(self):
972 972
973 973 readUnitConfObj = self.getReadUnitObj()
974 974
975 975 return readUnitConfObj.id
976 976
977 977 def getReadUnitObj(self):
978 978
979 979 for obj in self.procUnitConfObjDict.values():
980 980 if obj.getElementName() == "ReadUnit":
981 981 return obj
982 982
983 983 return None
984 984
985 985 def getProcUnitObj(self, id=None, name=None):
986 986
987 987 if id != None:
988 988 return self.procUnitConfObjDict[id]
989 989
990 990 if name != None:
991 991 return self.getProcUnitObjByName(name)
992 992
993 993 return None
994 994
995 995 def getProcUnitObjByName(self, name):
996 996
997 997 for obj in self.procUnitConfObjDict.values():
998 998 if obj.name == name:
999 999 return obj
1000 1000
1001 1001 return None
1002 1002
1003 1003 def procUnitItems(self):
1004 1004
1005 1005 return self.procUnitConfObjDict.items()
1006 1006
1007 1007 def makeXml(self):
1008 1008
1009 1009 projectElement = Element('Project')
1010 1010 projectElement.set('id', str(self.id))
1011 1011 projectElement.set('name', self.name)
1012 1012 projectElement.set('description', self.description)
1013 1013
1014 1014 for procUnitConfObj in self.procUnitConfObjDict.values():
1015 1015 procUnitConfObj.makeXml(projectElement)
1016 1016
1017 1017 self.projectElement = projectElement
1018 1018
1019 1019 def writeXml(self, filename=None):
1020 1020
1021 1021 if filename == None:
1022 1022 if self.filename:
1023 1023 filename = self.filename
1024 1024 else:
1025 1025 filename = "schain.xml"
1026 1026
1027 1027 if not filename:
1028 1028 print "filename has not been defined. Use setFilename(filename) for do it."
1029 1029 return 0
1030 1030
1031 1031 abs_file = os.path.abspath(filename)
1032 1032
1033 1033 if not os.access(os.path.dirname(abs_file), os.W_OK):
1034 1034 print "No write permission on %s" %os.path.dirname(abs_file)
1035 1035 return 0
1036 1036
1037 1037 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1038 1038 print "File %s already exists and it could not be overwriten" %abs_file
1039 1039 return 0
1040 1040
1041 1041 self.makeXml()
1042 1042
1043 1043 ElementTree(self.projectElement).write(abs_file, method='xml')
1044 1044
1045 1045 self.filename = abs_file
1046 1046
1047 1047 return 1
1048 1048
1049 1049 def readXml(self, filename = None):
1050 1050
1051 1051 if not filename:
1052 1052 print "filename is not defined"
1053 1053 return 0
1054 1054
1055 1055 abs_file = os.path.abspath(filename)
1056 1056
1057 1057 if not os.path.isfile(abs_file):
1058 1058 print "%s file does not exist" %abs_file
1059 1059 return 0
1060 1060
1061 1061 self.projectElement = None
1062 1062 self.procUnitConfObjDict = {}
1063 1063
1064 1064 try:
1065 1065 self.projectElement = ElementTree().parse(abs_file)
1066 1066 except:
1067 1067 print "Error reading %s, verify file format" %filename
1068 1068 return 0
1069 1069
1070 1070 self.project = self.projectElement.tag
1071 1071
1072 1072 self.id = self.projectElement.get('id')
1073 1073 self.name = self.projectElement.get('name')
1074 1074 self.description = self.projectElement.get('description')
1075 1075
1076 1076 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1077 1077
1078 1078 for readUnitElement in readUnitElementList:
1079 1079 readUnitConfObj = ReadUnitConf()
1080 1080 readUnitConfObj.readXml(readUnitElement)
1081 1081
1082 1082 if readUnitConfObj.parentId == None:
1083 1083 readUnitConfObj.parentId = self.id
1084 1084
1085 1085 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1086 1086
1087 1087 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1088 1088
1089 1089 for procUnitElement in procUnitElementList:
1090 1090 procUnitConfObj = ProcUnitConf()
1091 1091 procUnitConfObj.readXml(procUnitElement)
1092 1092
1093 1093 if procUnitConfObj.parentId == None:
1094 1094 procUnitConfObj.parentId = self.id
1095 1095
1096 1096 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
1097 1097
1098 1098 self.filename = abs_file
1099 1099
1100 1100 return 1
1101 1101
1102 1102 def printattr(self):
1103 1103
1104 1104 print "Project[%s]: name = %s, description = %s" %(self.id,
1105 1105 self.name,
1106 1106 self.description)
1107 1107
1108 1108 for procUnitConfObj in self.procUnitConfObjDict.values():
1109 1109 procUnitConfObj.printattr()
1110 1110
1111 1111 def createObjects(self):
1112 1112
1113 1113 for procUnitConfObj in self.procUnitConfObjDict.values():
1114 1114 procUnitConfObj.createObjects(self.plotterQueue)
1115 1115
1116 1116 def __connect(self, objIN, thisObj):
1117 1117
1118 1118 thisObj.setInput(objIN.getOutputObj())
1119 1119
1120 1120 def connectObjects(self):
1121 1121
1122 1122 for thisPUConfObj in self.procUnitConfObjDict.values():
1123 1123
1124 1124 inputId = thisPUConfObj.getInputId()
1125 1125
1126 1126 if int(inputId) == 0:
1127 1127 continue
1128 1128
1129 1129 #Get input object
1130 1130 puConfINObj = self.procUnitConfObjDict[inputId]
1131 1131 puObjIN = puConfINObj.getProcUnitObj()
1132 1132
1133 1133 #Get current object
1134 1134 thisPUObj = thisPUConfObj.getProcUnitObj()
1135 1135
1136 1136 self.__connect(puObjIN, thisPUObj)
1137 1137
1138 1138 def __handleError(self, procUnitConfObj, send_email=True):
1139 1139
1140 1140 import socket
1141 1141
1142 1142 err = traceback.format_exception(sys.exc_info()[0],
1143 1143 sys.exc_info()[1],
1144 1144 sys.exc_info()[2])
1145 1145
1146 1146 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1147 1147 print "***** %s" %err[-1]
1148 1148
1149 1149 message = "".join(err)
1150 1150
1151 1151 sys.stderr.write(message)
1152 1152
1153 1153 if not send_email:
1154 1154 return
1155 1155
1156 1156 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1157 1157
1158 1158 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1159 1159 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1160 1160 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1161 1161 subtitle += "Configuration file: %s\n" %self.filename
1162 1162 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1163 1163
1164 1164 readUnitConfObj = self.getReadUnitObj()
1165 1165 if readUnitConfObj:
1166 1166 subtitle += "\nInput parameters:\n"
1167 1167 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1168 1168 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1169 1169 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1170 1170 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1171 1171 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1172 1172 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1173 1173
1174 1174 adminObj = schainpy.admin.SchainNotify()
1175 1175 adminObj.sendAlert(message=message,
1176 1176 subject=subject,
1177 1177 subtitle=subtitle,
1178 1178 filename=self.filename)
1179 1179
1180 1180 def isPaused(self):
1181 1181 return 0
1182 1182
1183 1183 def isStopped(self):
1184 1184 return 0
1185 1185
1186 1186 def runController(self):
1187 1187 """
1188 1188 returns 0 when this process has been stopped, 1 otherwise
1189 1189 """
1190 1190
1191 1191 if self.isPaused():
1192 1192 print "Process suspended"
1193 1193
1194 1194 while True:
1195 1195 sleep(0.1)
1196 1196
1197 1197 if not self.isPaused():
1198 1198 break
1199 1199
1200 1200 if self.isStopped():
1201 1201 break
1202 1202
1203 1203 print "Process reinitialized"
1204 1204
1205 1205 if self.isStopped():
1206 1206 print "Process stopped"
1207 1207 return 0
1208 1208
1209 1209 return 1
1210 1210
1211 1211 def setFilename(self, filename):
1212 1212
1213 1213 self.filename = filename
1214 1214
1215 1215 def setPlotterQueue(self, plotter_queue):
1216 1216
1217 1217 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1218 1218
1219 1219 def getPlotterQueue(self):
1220 1220
1221 1221 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1222 1222
1223 1223 def useExternalPlotter(self):
1224 1224
1225 1225 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1226 1226
1227 1227
1228 1228 def run(self, filename=None):
1229 1229
1230 1230 # self.writeXml(filename)
1231 1231 self.createObjects()
1232 1232 self.connectObjects()
1233 1233
1234 1234 print
1235 1235 print "*"*60
1236 1236 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1237 1237 print "*"*60
1238 1238 print
1239 1239
1240 1240 keyList = self.procUnitConfObjDict.keys()
1241 1241 keyList.sort()
1242 1242
1243 1243 while(True):
1244 1244
1245 1245 is_ok = False
1246 1246
1247 1247 for procKey in keyList:
1248 1248 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1249 1249
1250 1250 procUnitConfObj = self.procUnitConfObjDict[procKey]
1251 1251
1252 1252 try:
1253 1253 sts = procUnitConfObj.run()
1254 1254 is_ok = is_ok or sts
1255 1255 except KeyboardInterrupt:
1256 1256 is_ok = False
1257 1257 break
1258 1258 except ValueError, e:
1259 1259 sleep(0.5)
1260 1260 self.__handleError(procUnitConfObj, send_email=True)
1261 1261 is_ok = False
1262 1262 break
1263 1263 except:
1264 1264 sleep(0.5)
1265 1265 self.__handleError(procUnitConfObj)
1266 1266 is_ok = False
1267 1267 break
1268 1268
1269 1269 #If every process unit finished so end process
1270 1270 if not(is_ok):
1271 1271 # print "Every process unit have finished"
1272 1272 break
1273 1273
1274 1274 if not self.runController():
1275 1275 break
1276 1276
1277 1277 #Closing every process
1278 1278 for procKey in keyList:
1279 1279 procUnitConfObj = self.procUnitConfObjDict[procKey]
1280 1280 procUnitConfObj.close()
Show file before | Show file after | Hide comments
@@ -1,1855 +1,1854
1 1 '''
2 2 Created on Jul 2, 2014
3 3
4 4 @author: roj-idl71
5 5 '''
6 6 import os
7 7 import sys
8 8 import glob
9 9 import time
10 10 import numpy
11 11 import fnmatch
12 12 import inspect
13 13 import time, datetime
14 14 import traceback
15 15 import zmq
16 16
17 17 try:
18 18 from gevent import sleep
19 19 except:
20 20 from time import sleep
21 21
22 22 from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
23 23 from schainpy.model.data.jroheaderIO import get_dtype_index, get_numpy_dtype, get_procflag_dtype, get_dtype_width
24 24
25 25 LOCALTIME = True
26 26
27 27 def isNumber(cad):
28 28 """
29 29 Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
30 30
31 31 Excepciones:
32 32 Si un determinado string no puede ser convertido a numero
33 33 Input:
34 34 str, string al cual se le analiza para determinar si convertible a un numero o no
35 35
36 36 Return:
37 37 True : si el string es uno numerico
38 38 False : no es un string numerico
39 39 """
40 40 try:
41 41 float( cad )
42 42 return True
43 43 except:
44 44 return False
45 45
46 46 def isFileInEpoch(filename, startUTSeconds, endUTSeconds):
47 47 """
48 48 Esta funcion determina si un archivo de datos se encuentra o no dentro del rango de fecha especificado.
49 49
50 50 Inputs:
51 51 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
52 52
53 53 startUTSeconds : fecha inicial del rango seleccionado. La fecha esta dada en
54 54 segundos contados desde 01/01/1970.
55 55 endUTSeconds : fecha final del rango seleccionado. La fecha esta dada en
56 56 segundos contados desde 01/01/1970.
57 57
58 58 Return:
59 59 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
60 60 fecha especificado, de lo contrario retorna False.
61 61
62 62 Excepciones:
63 63 Si el archivo no existe o no puede ser abierto
64 64 Si la cabecera no puede ser leida.
65 65
66 66 """
67 67 basicHeaderObj = BasicHeader(LOCALTIME)
68 68
69 69 try:
70 70 fp = open(filename,'rb')
71 71 except IOError:
72 72 print "The file %s can't be opened" %(filename)
73 73 return 0
74 74
75 75 sts = basicHeaderObj.read(fp)
76 76 fp.close()
77 77
78 78 if not(sts):
79 79 print "Skipping the file %s because it has not a valid header" %(filename)
80 80 return 0
81 81
82 82 if not ((startUTSeconds <= basicHeaderObj.utc) and (endUTSeconds > basicHeaderObj.utc)):
83 83 return 0
84 84
85 85 return 1
86 86
87 87 def isTimeInRange(thisTime, startTime, endTime):
88 88
89 89 if endTime >= startTime:
90 90 if (thisTime < startTime) or (thisTime > endTime):
91 91 return 0
92 92
93 93 return 1
94 94 else:
95 95 if (thisTime < startTime) and (thisTime > endTime):
96 96 return 0
97 97
98 98 return 1
99 99
100 100 def isFileInTimeRange(filename, startDate, endDate, startTime, endTime):
101 101 """
102 102 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
103 103
104 104 Inputs:
105 105 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
106 106
107 107 startDate : fecha inicial del rango seleccionado en formato datetime.date
108 108
109 109 endDate : fecha final del rango seleccionado en formato datetime.date
110 110
111 111 startTime : tiempo inicial del rango seleccionado en formato datetime.time
112 112
113 113 endTime : tiempo final del rango seleccionado en formato datetime.time
114 114
115 115 Return:
116 116 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
117 117 fecha especificado, de lo contrario retorna False.
118 118
119 119 Excepciones:
120 120 Si el archivo no existe o no puede ser abierto
121 121 Si la cabecera no puede ser leida.
122 122
123 123 """
124 124
125 125
126 126 try:
127 127 fp = open(filename,'rb')
128 128 except IOError:
129 129 print "The file %s can't be opened" %(filename)
130 130 return None
131 131
132 132 firstBasicHeaderObj = BasicHeader(LOCALTIME)
133 133 systemHeaderObj = SystemHeader()
134 134 radarControllerHeaderObj = RadarControllerHeader()
135 135 processingHeaderObj = ProcessingHeader()
136 136
137 137 lastBasicHeaderObj = BasicHeader(LOCALTIME)
138 138
139 139 sts = firstBasicHeaderObj.read(fp)
140 140
141 141 if not(sts):
142 142 print "[Reading] Skipping the file %s because it has not a valid header" %(filename)
143 143 return None
144 144
145 145 if not systemHeaderObj.read(fp):
146 146 return None
147 147
148 148 if not radarControllerHeaderObj.read(fp):
149 149 return None
150 150
151 151 if not processingHeaderObj.read(fp):
152 152 return None
153 153
154 154 filesize = os.path.getsize(filename)
155 155
156 156 offset = processingHeaderObj.blockSize + 24 #header size
157 157
158 158 if filesize <= offset:
159 159 print "[Reading] %s: This file has not enough data" %filename
160 160 return None
161 161
162 162 fp.seek(-offset, 2)
163 163
164 164 sts = lastBasicHeaderObj.read(fp)
165 165
166 166 fp.close()
167 167
168 168 thisDatetime = lastBasicHeaderObj.datatime
169 169 thisTime_last_block = thisDatetime.time()
170 170
171 171 thisDatetime = firstBasicHeaderObj.datatime
172 172 thisDate = thisDatetime.date()
173 173 thisTime_first_block = thisDatetime.time()
174 174
175 175 #General case
176 176 # o>>>>>>>>>>>>>><<<<<<<<<<<<<<o
177 177 #-----------o----------------------------o-----------
178 178 # startTime endTime
179 179
180 180 if endTime >= startTime:
181 181 if (thisTime_last_block < startTime) or (thisTime_first_block > endTime):
182 182 return None
183 183
184 184 return thisDatetime
185 185
186 186 #If endTime < startTime then endTime belongs to the next day
187 187
188 188
189 189 #<<<<<<<<<<<o o>>>>>>>>>>>
190 190 #-----------o----------------------------o-----------
191 191 # endTime startTime
192 192
193 193 if (thisDate == startDate) and (thisTime_last_block < startTime):
194 194 return None
195 195
196 196 if (thisDate == endDate) and (thisTime_first_block > endTime):
197 197 return None
198 198
199 199 if (thisTime_last_block < startTime) and (thisTime_first_block > endTime):
200 200 return None
201 201
202 202 return thisDatetime
203 203
204 204 def isFolderInDateRange(folder, startDate=None, endDate=None):
205 205 """
206 206 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
207 207
208 208 Inputs:
209 209 folder : nombre completo del directorio.
210 210 Su formato deberia ser "/path_root/?YYYYDDD"
211 211
212 212 siendo:
213 213 YYYY : Anio (ejemplo 2015)
214 214 DDD : Dia del anio (ejemplo 305)
215 215
216 216 startDate : fecha inicial del rango seleccionado en formato datetime.date
217 217
218 218 endDate : fecha final del rango seleccionado en formato datetime.date
219 219
220 220 Return:
221 221 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
222 222 fecha especificado, de lo contrario retorna False.
223 223 Excepciones:
224 224 Si el directorio no tiene el formato adecuado
225 225 """
226 226
227 227 basename = os.path.basename(folder)
228 228
229 229 if not isRadarFolder(basename):
230 230 print "The folder %s has not the rigth format" %folder
231 231 return 0
232 232
233 233 if startDate and endDate:
234 234 thisDate = getDateFromRadarFolder(basename)
235 235
236 236 if thisDate < startDate:
237 237 return 0
238 238
239 239 if thisDate > endDate:
240 240 return 0
241 241
242 242 return 1
243 243
244 244 def isFileInDateRange(filename, startDate=None, endDate=None):
245 245 """
246 246 Retorna 1 si el archivo de datos se encuentra dentro del rango de horas especificado.
247 247
248 248 Inputs:
249 249 filename : nombre completo del archivo de datos en formato Jicamarca (.r)
250 250
251 251 Su formato deberia ser "?YYYYDDDsss"
252 252
253 253 siendo:
254 254 YYYY : Anio (ejemplo 2015)
255 255 DDD : Dia del anio (ejemplo 305)
256 256 sss : set
257 257
258 258 startDate : fecha inicial del rango seleccionado en formato datetime.date
259 259
260 260 endDate : fecha final del rango seleccionado en formato datetime.date
261 261
262 262 Return:
263 263 Boolean : Retorna True si el archivo de datos contiene datos en el rango de
264 264 fecha especificado, de lo contrario retorna False.
265 265 Excepciones:
266 266 Si el archivo no tiene el formato adecuado
267 267 """
268 268
269 269 basename = os.path.basename(filename)
270 270
271 271 if not isRadarFile(basename):
272 272 print "The filename %s has not the rigth format" %filename
273 273 return 0
274 274
275 275 if startDate and endDate:
276 276 thisDate = getDateFromRadarFile(basename)
277 277
278 278 if thisDate < startDate:
279 279 return 0
280 280
281 281 if thisDate > endDate:
282 282 return 0
283 283
284 284 return 1
285 285
286 286 def getFileFromSet(path, ext, set):
287 287 validFilelist = []
288 288 fileList = os.listdir(path)
289 289
290 290 # 0 1234 567 89A BCDE
291 291 # H YYYY DDD SSS .ext
292 292
293 293 for thisFile in fileList:
294 294 try:
295 295 year = int(thisFile[1:5])
296 296 doy = int(thisFile[5:8])
297 297 except:
298 298 continue
299 299
300 300 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
301 301 continue
302 302
303 303 validFilelist.append(thisFile)
304 304
305 305 myfile = fnmatch.filter(validFilelist,'*%4.4d%3.3d%3.3d*'%(year,doy,set))
306 306
307 307 if len(myfile)!= 0:
308 308 return myfile[0]
309 309 else:
310 310 filename = '*%4.4d%3.3d%3.3d%s'%(year,doy,set,ext.lower())
311 311 print 'the filename %s does not exist'%filename
312 312 print '...going to the last file: '
313 313
314 314 if validFilelist:
315 315 validFilelist = sorted( validFilelist, key=str.lower )
316 316 return validFilelist[-1]
317 317
318 318 return None
319 319
320 320 def getlastFileFromPath(path, ext):
321 321 """
322 322 Depura el fileList dejando solo los que cumplan el formato de "PYYYYDDDSSS.ext"
323 323 al final de la depuracion devuelve el ultimo file de la lista que quedo.
324 324
325 325 Input:
326 326 fileList : lista conteniendo todos los files (sin path) que componen una determinada carpeta
327 327 ext : extension de los files contenidos en una carpeta
328 328
329 329 Return:
330 330 El ultimo file de una determinada carpeta, no se considera el path.
331 331 """
332 332 validFilelist = []
333 333 fileList = os.listdir(path)
334 334
335 335 # 0 1234 567 89A BCDE
336 336 # H YYYY DDD SSS .ext
337 337
338 338 for thisFile in fileList:
339 339
340 340 year = thisFile[1:5]
341 341 if not isNumber(year):
342 342 continue
343 343
344 344 doy = thisFile[5:8]
345 345 if not isNumber(doy):
346 346 continue
347 347
348 348 year = int(year)
349 349 doy = int(doy)
350 350
351 351 if (os.path.splitext(thisFile)[-1].lower() != ext.lower()):
352 352 continue
353 353
354 354 validFilelist.append(thisFile)
355 355
356 356 if validFilelist:
357 357 validFilelist = sorted( validFilelist, key=str.lower )
358 358 return validFilelist[-1]
359 359
360 360 return None
361 361
362 362 def checkForRealPath(path, foldercounter, year, doy, set, ext):
363 363 """
364 364 Por ser Linux Case Sensitive entonces checkForRealPath encuentra el nombre correcto de un path,
365 365 Prueba por varias combinaciones de nombres entre mayusculas y minusculas para determinar
366 366 el path exacto de un determinado file.
367 367
368 368 Example :
369 369 nombre correcto del file es .../.../D2009307/P2009307367.ext
370 370
371 371 Entonces la funcion prueba con las siguientes combinaciones
372 372 .../.../y2009307367.ext
373 373 .../.../Y2009307367.ext
374 374 .../.../x2009307/y2009307367.ext
375 375 .../.../x2009307/Y2009307367.ext
376 376 .../.../X2009307/y2009307367.ext
377 377 .../.../X2009307/Y2009307367.ext
378 378 siendo para este caso, la ultima combinacion de letras, identica al file buscado
379 379
380 380 Return:
381 381 Si encuentra la cobinacion adecuada devuelve el path completo y el nombre del file
382 382 caso contrario devuelve None como path y el la ultima combinacion de nombre en mayusculas
383 383 para el filename
384 384 """
385 385 fullfilename = None
386 386 find_flag = False
387 387 filename = None
388 388
389 389 prefixDirList = [None,'d','D']
390 390 if ext.lower() == ".r": #voltage
391 391 prefixFileList = ['d','D']
392 392 elif ext.lower() == ".pdata": #spectra
393 393 prefixFileList = ['p','P']
394 394 else:
395 395 return None, filename
396 396
397 397 #barrido por las combinaciones posibles
398 398 for prefixDir in prefixDirList:
399 399 thispath = path
400 400 if prefixDir != None:
401 401 #formo el nombre del directorio xYYYYDDD (x=d o x=D)
402 402 if foldercounter == 0:
403 403 thispath = os.path.join(path, "%s%04d%03d" % ( prefixDir, year, doy ))
404 404 else:
405 405 thispath = os.path.join(path, "%s%04d%03d_%02d" % ( prefixDir, year, doy , foldercounter))
406 406 for prefixFile in prefixFileList: #barrido por las dos combinaciones posibles de "D"
407 407 filename = "%s%04d%03d%03d%s" % ( prefixFile, year, doy, set, ext ) #formo el nombre del file xYYYYDDDSSS.ext
408 408 fullfilename = os.path.join( thispath, filename ) #formo el path completo
409 409
410 410 if os.path.exists( fullfilename ): #verifico que exista
411 411 find_flag = True
412 412 break
413 413 if find_flag:
414 414 break
415 415
416 416 if not(find_flag):
417 417 return None, filename
418 418
419 419 return fullfilename, filename
420 420
421 421 def isRadarFolder(folder):
422 422 try:
423 423 year = int(folder[1:5])
424 424 doy = int(folder[5:8])
425 425 except:
426 426 return 0
427 427
428 428 return 1
429 429
430 430 def isRadarFile(file):
431 431 try:
432 432 year = int(file[1:5])
433 433 doy = int(file[5:8])
434 434 set = int(file[8:11])
435 435 except:
436 436 return 0
437 437
438 438 return 1
439 439
440 440 def getDateFromRadarFile(file):
441 441 try:
442 442 year = int(file[1:5])
443 443 doy = int(file[5:8])
444 444 set = int(file[8:11])
445 445 except:
446 446 return None
447 447
448 448 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
449 449 return thisDate
450 450
451 451 def getDateFromRadarFolder(folder):
452 452 try:
453 453 year = int(folder[1:5])
454 454 doy = int(folder[5:8])
455 455 except:
456 456 return None
457 457
458 458 thisDate = datetime.date(year, 1, 1) + datetime.timedelta(doy-1)
459 459 return thisDate
460 460
461 461 class JRODataIO:
462 462
463 463 c = 3E8
464 464
465 465 isConfig = False
466 466
467 467 basicHeaderObj = None
468 468
469 469 systemHeaderObj = None
470 470
471 471 radarControllerHeaderObj = None
472 472
473 473 processingHeaderObj = None
474 474
475 475 dtype = None
476 476
477 477 pathList = []
478 478
479 479 filenameList = []
480 480
481 481 filename = None
482 482
483 483 ext = None
484 484
485 485 flagIsNewFile = 1
486 486
487 487 flagDiscontinuousBlock = 0
488 488
489 489 flagIsNewBlock = 0
490 490
491 491 fp = None
492 492
493 493 firstHeaderSize = 0
494 494
495 495 basicHeaderSize = 24
496 496
497 497 versionFile = 1103
498 498
499 499 fileSize = None
500 500
501 501 # ippSeconds = None
502 502
503 503 fileSizeByHeader = None
504 504
505 505 fileIndex = None
506 506
507 507 profileIndex = None
508 508
509 509 blockIndex = None
510 510
511 511 nTotalBlocks = None
512 512
513 513 maxTimeStep = 30
514 514
515 515 lastUTTime = None
516 516
517 517 datablock = None
518 518
519 519 dataOut = None
520 520
521 521 blocksize = None
522 522
523 523 getByBlock = False
524 524
525 525 def __init__(self):
526 526
527 527 raise NotImplementedError
528 528
529 529 def run(self):
530 530
531 531 raise NotImplementedError
532 532
533 533 def getDtypeWidth(self):
534 534
535 535 dtype_index = get_dtype_index(self.dtype)
536 536 dtype_width = get_dtype_width(dtype_index)
537 537
538 538 return dtype_width
539 539
540 540 def getAllowedArgs(self):
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544 544
545 545 firstTime = True
546 546 online = 0
547 547
548 548 realtime = 0
549 549
550 550 nReadBlocks = 0
551 551
552 552 delay = 10 #number of seconds waiting a new file
553 553
554 554 nTries = 3 #quantity tries
555 555
556 556 nFiles = 3 #number of files for searching
557 557
558 558 path = None
559 559
560 560 foldercounter = 0
561 561
562 562 flagNoMoreFiles = 0
563 563
564 564 datetimeList = []
565 565
566 566 __isFirstTimeOnline = 1
567 567
568 568 __printInfo = True
569 569
570 570 profileIndex = None
571 571
572 572 nTxs = 1
573 573
574 574 txIndex = None
575 575
576 576 #Added--------------------
577 577
578 578 selBlocksize = None
579 579
580 580 selBlocktime = None
581 581
582 582 onlineWithDate = False
583 583 def __init__(self):
584 584
585 585 """
586 586 This class is used to find data files
587 587
588 588 Example:
589 589 reader = JRODataReader()
590 590 fileList = reader.findDataFiles()
591 591
592 592 """
593 593 pass
594 594
595 595
596 596 def createObjByDefault(self):
597 597 """
598 598
599 599 """
600 600 raise NotImplementedError
601 601
602 602 def getBlockDimension(self):
603 603
604 604 raise NotImplementedError
605 605
606 606 def __searchFilesOffLine(self,
607 607 path,
608 608 startDate=None,
609 609 endDate=None,
610 610 startTime=datetime.time(0,0,0),
611 611 endTime=datetime.time(23,59,59),
612 612 set=None,
613 613 expLabel='',
614 614 ext='.r',
615 615 queue=None,
616 616 cursor=None,
617 617 skip=None,
618 618 walk=True):
619 619 self.filenameList = []
620 620 self.datetimeList = []
621 621
622 622 pathList = []
623 623
624 624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 625
626 626 if dateList == []:
627 627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 628 return None, None
629 629
630 630 if len(dateList) > 1:
631 631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
632 632 else:
633 633 print "[Reading] Data found for date range [%s - %s]: date = %s" %(startDate, endDate, dateList[0])
634 634
635 635 filenameList = []
636 636 datetimeList = []
637 637
638 638 for thisPath in pathList:
639 639 # thisPath = pathList[pathDict[file]]
640 640
641 641 fileList = glob.glob1(thisPath, "*%s" %ext)
642 642 fileList.sort()
643 643
644 644 skippedFileList = []
645 645
646 646 if cursor is not None and skip is not None:
647 647 # if cursor*skip > len(fileList):
648 648 if skip == 0:
649 649 if queue is not None:
650 650 queue.put(len(fileList))
651 651 skippedFileList = []
652 652 else:
653 653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
654 654
655 655 else:
656 656 skippedFileList = fileList
657 657
658 658 for file in skippedFileList:
659 659
660 660 filename = os.path.join(thisPath,file)
661 661
662 662 if not isFileInDateRange(filename, startDate, endDate):
663 663 continue
664 664
665 665 thisDatetime = isFileInTimeRange(filename, startDate, endDate, startTime, endTime)
666 666
667 667 if not(thisDatetime):
668 668 continue
669 669
670 670 filenameList.append(filename)
671 671 datetimeList.append(thisDatetime)
672 672
673 673 if not(filenameList):
674 674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 675 return None, None
676 676
677 677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 678 print
679 679
680 680 for i in range(len(filenameList)):
681 681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 682
683 683 self.filenameList = filenameList
684 684 self.datetimeList = datetimeList
685 685 return pathList, filenameList
686 686
687 687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
688 688
689 689 """
690 690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 691 devuelve el archivo encontrado ademas de otros datos.
692 692
693 693 Input:
694 694 path : carpeta donde estan contenidos los files que contiene data
695 695
696 696 expLabel : Nombre del subexperimento (subfolder)
697 697
698 698 ext : extension de los files
699 699
700 700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
701 701
702 702 Return:
703 703 directory : eL directorio donde esta el file encontrado
704 704 filename : el ultimo file de una determinada carpeta
705 705 year : el anho
706 706 doy : el numero de dia del anho
707 707 set : el set del archivo
708 708
709 709
710 710 """
711 711 pathList = None
712 712 filenameList = None
713 713 if not os.path.isdir(path):
714 714 return None, None, None, None, None, None
715 715
716 716 dirList = []
717 717
718 718 if not walk:
719 719 fullpath = path
720 720 foldercounter = 0
721 721 else:
722 722 # Filtra solo los directorios
723 723 for thisPath in os.listdir(path):
724 724 if not os.path.isdir(os.path.join(path,thisPath)):
725 725 continue
726 726 if not isRadarFolder(thisPath):
727 727 continue
728 728
729 729 dirList.append(thisPath)
730 730
731 731 if not(dirList):
732 732 return None, None, None, None, None, None
733 733
734 734 dirList = sorted( dirList, key=str.lower )
735 735
736 736 doypath = dirList[-1]
737 737 foldercounter = int(doypath.split('_')[1]) if len(doypath.split('_'))>1 else 0
738 738 fullpath = os.path.join(path, doypath, expLabel)
739 739
740 740
741 741 print "[Reading] %s folder was found: " %(fullpath )
742 742
743 743 if set == None:
744 744 filename = getlastFileFromPath(fullpath, ext)
745 745 else:
746 746 filename = getFileFromSet(fullpath, ext, set)
747 747
748 748 if not(filename):
749 749 return None, None, None, None, None, None
750 750
751 751 print "[Reading] %s file was found" %(filename)
752 752
753 753 if not(self.__verifyFile(os.path.join(fullpath, filename))):
754 754 return None, None, None, None, None, None
755 755
756 756 year = int( filename[1:5] )
757 757 doy = int( filename[5:8] )
758 758 set = int( filename[8:11] )
759 759
760 760 return fullpath, foldercounter, filename, year, doy, set
761 761
762 762 def __setNextFileOffline(self):
763 763
764 764 idFile = self.fileIndex
765 765
766 766 while (True):
767 767 idFile += 1
768 768 if not(idFile < len(self.filenameList)):
769 769 self.flagNoMoreFiles = 1
770 770 # print "[Reading] No more Files"
771 771 return 0
772 772
773 773 filename = self.filenameList[idFile]
774 774
775 775 if not(self.__verifyFile(filename)):
776 776 continue
777 777
778 778 fileSize = os.path.getsize(filename)
779 779 fp = open(filename,'rb')
780 780 break
781 781
782 782 self.flagIsNewFile = 1
783 783 self.fileIndex = idFile
784 784 self.filename = filename
785 785 self.fileSize = fileSize
786 786 self.fp = fp
787 787
788 788 #print "[Reading] Setting the file: %s"%self.filename
789 789
790 790 return 1
791 791
792 792 def __setNextFileOnline(self):
793 793 """
794 794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 796 siguientes.
797 797
798 798 Affected:
799 799 self.flagIsNewFile
800 800 self.filename
801 801 self.fileSize
802 802 self.fp
803 803 self.set
804 804 self.flagNoMoreFiles
805 805
806 806 Return:
807 807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 808 1 : si el file fue abierto con exito y esta listo a ser leido
809 809
810 810 Excepciones:
811 811 Si un determinado file no puede ser abierto
812 812 """
813 813
814 814 nFiles = 0
815 815 fileOk_flag = False
816 816 firstTime_flag = True
817 817
818 818 self.set += 1
819 819
820 820 if self.set > 999:
821 821 self.set = 0
822 822 self.foldercounter += 1
823 823
824 824 #busca el 1er file disponible
825 825 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
826 826 if fullfilename:
827 827 if self.__verifyFile(fullfilename, False):
828 828 fileOk_flag = True
829 829
830 830 #si no encuentra un file entonces espera y vuelve a buscar
831 831 if not(fileOk_flag):
832 832 for nFiles in range(self.nFiles+1): #busco en los siguientes self.nFiles+1 files posibles
833 833
834 834 if firstTime_flag: #si es la 1era vez entonces hace el for self.nTries veces
835 835 tries = self.nTries
836 836 else:
837 837 tries = 1 #si no es la 1era vez entonces solo lo hace una vez
838 838
839 839 for nTries in range( tries ):
840 840 if firstTime_flag:
841 841 print "\t[Reading] Waiting %0.2f sec for the next file: \"%s\" , try %03d ..." % ( self.delay, filename, nTries+1 )
842 842 sleep( self.delay )
843 843 else:
844 844 print "\t[Reading] Searching the next \"%s%04d%03d%03d%s\" file ..." % (self.optchar, self.year, self.doy, self.set, self.ext)
845 845
846 846 fullfilename, filename = checkForRealPath( self.path, self.foldercounter, self.year, self.doy, self.set, self.ext )
847 847 if fullfilename:
848 848 if self.__verifyFile(fullfilename):
849 849 fileOk_flag = True
850 850 break
851 851
852 852 if fileOk_flag:
853 853 break
854 854
855 855 firstTime_flag = False
856 856
857 857 print "\t[Reading] Skipping the file \"%s\" due to this file doesn't exist" % filename
858 858 self.set += 1
859 859
860 860 if nFiles == (self.nFiles-1): #si no encuentro el file buscado cambio de carpeta y busco en la siguiente carpeta
861 861 self.set = 0
862 862 self.doy += 1
863 863 self.foldercounter = 0
864 864
865 865 if fileOk_flag:
866 866 self.fileSize = os.path.getsize( fullfilename )
867 867 self.filename = fullfilename
868 868 self.flagIsNewFile = 1
869 869 if self.fp != None: self.fp.close()
870 870 self.fp = open(fullfilename, 'rb')
871 871 self.flagNoMoreFiles = 0
872 872 # print '[Reading] Setting the file: %s' % fullfilename
873 873 else:
874 874 self.fileSize = 0
875 875 self.filename = None
876 876 self.flagIsNewFile = 0
877 877 self.fp = None
878 878 self.flagNoMoreFiles = 1
879 879 # print '[Reading] No more files to read'
880 880
881 881 return fileOk_flag
882 882
883 883 def setNextFile(self):
884 884 if self.fp != None:
885 885 self.fp.close()
886 886 if self.online:
887 887 newFile = self.__setNextFileOnline()
888 888 else:
889 889 newFile = self.__setNextFileOffline()
890 890 if not(newFile):
891 891 if self.onlineWithDate is True:
892 892 self.onlineWithDate=False
893 893 self.online = True
894 894 self.firstTime = False
895 895 self.setup(
896 896 path=self.path,
897 897 startDate=self.startDate,
898 898 endDate=self.endDate,
899 899 startTime=self.startTime ,
900 900 endTime=self.endTime,
901 901 set=self.set,
902 902 expLabel=self.expLabel,
903 903 ext=self.ext,
904 904 online=self.online,
905 905 delay=self.delay,
906 906 walk=self.walk,
907 907 getblock=self.getblock,
908 908 nTxs=self.nTxs,
909 909 realtime=self.realtime,
910 910 blocksize=self.blocksize,
911 911 blocktime=self.blocktime
912 912 )
913 913 return 1
914 914 print '[Reading] No more files to read'
915 915 return 0
916 916
917 917 if self.verbose:
918 918 print '[Reading] Setting the file: %s' % self.filename
919 919
920 920 self.__readFirstHeader()
921 921 self.nReadBlocks = 0
922 922 return 1
923 923
924 924 def __waitNewBlock(self):
925 925 """
926 926 Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
927 927
928 928 Si el modo de lectura es OffLine siempre retorn 0
929 929 """
930 930 if not self.online:
931 931 return 0
932 932
933 933 if (self.nReadBlocks >= self.processingHeaderObj.dataBlocksPerFile):
934 934 return 0
935 935
936 936 currentPointer = self.fp.tell()
937 937
938 938 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
939 939
940 940 for nTries in range( self.nTries ):
941 941
942 942 self.fp.close()
943 943 self.fp = open( self.filename, 'rb' )
944 944 self.fp.seek( currentPointer )
945 945
946 946 self.fileSize = os.path.getsize( self.filename )
947 947 currentSize = self.fileSize - currentPointer
948 948
949 949 if ( currentSize >= neededSize ):
950 950 self.basicHeaderObj.read(self.fp)
951 951 return 1
952 952
953 953 if self.fileSize == self.fileSizeByHeader:
954 954 # self.flagEoF = True
955 955 return 0
956 956
957 957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
958 958 sleep( self.delay )
959 959
960 960
961 961 return 0
962 962
963 963 def waitDataBlock(self,pointer_location):
964 964
965 965 currentPointer = pointer_location
966 966
967 967 neededSize = self.processingHeaderObj.blockSize #+ self.basicHeaderSize
968 968
969 969 for nTries in range( self.nTries ):
970 970 self.fp.close()
971 971 self.fp = open( self.filename, 'rb' )
972 972 self.fp.seek( currentPointer )
973 973
974 974 self.fileSize = os.path.getsize( self.filename )
975 975 currentSize = self.fileSize - currentPointer
976 976
977 977 if ( currentSize >= neededSize ):
978 978 return 1
979 979
980 980 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
981 981 sleep( self.delay )
982 982
983 983 return 0
984 984
985 985 def __jumpToLastBlock(self):
986 986
987 987 if not(self.__isFirstTimeOnline):
988 988 return
989 989
990 990 csize = self.fileSize - self.fp.tell()
991 991 blocksize = self.processingHeaderObj.blockSize
992 992
993 993 #salta el primer bloque de datos
994 994 if csize > self.processingHeaderObj.blockSize:
995 995 self.fp.seek(self.fp.tell() + blocksize)
996 996 else:
997 997 return
998 998
999 999 csize = self.fileSize - self.fp.tell()
1000 1000 neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1001 1001 while True:
1002 1002
1003 1003 if self.fp.tell()<self.fileSize:
1004 1004 self.fp.seek(self.fp.tell() + neededsize)
1005 1005 else:
1006 1006 self.fp.seek(self.fp.tell() - neededsize)
1007 1007 break
1008 1008
1009 1009 # csize = self.fileSize - self.fp.tell()
1010 1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 1011 # factor = int(csize/neededsize)
1012 1012 # if factor > 0:
1013 1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014 1014
1015 1015 self.flagIsNewFile = 0
1016 1016 self.__isFirstTimeOnline = 0
1017 1017
1018 1018 def __setNewBlock(self):
1019 1019 #if self.server is None:
1020 1020 if self.fp == None:
1021 1021 return 0
1022 1022
1023 1023 # if self.online:
1024 1024 # self.__jumpToLastBlock()
1025 1025
1026 1026 if self.flagIsNewFile:
1027 1027 self.lastUTTime = self.basicHeaderObj.utc
1028 1028 return 1
1029 1029
1030 1030 if self.realtime:
1031 1031 self.flagDiscontinuousBlock = 1
1032 1032 if not(self.setNextFile()):
1033 1033 return 0
1034 1034 else:
1035 1035 return 1
1036 1036 #if self.server is None:
1037 1037 currentSize = self.fileSize - self.fp.tell()
1038 1038 neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1039 1039 if (currentSize >= neededSize):
1040 1040 self.basicHeaderObj.read(self.fp)
1041 1041 self.lastUTTime = self.basicHeaderObj.utc
1042 1042 return 1
1043 1043 # else:
1044 1044 # self.basicHeaderObj.read(self.zHeader)
1045 1045 # self.lastUTTime = self.basicHeaderObj.utc
1046 1046 # return 1
1047 1047 if self.__waitNewBlock():
1048 1048 self.lastUTTime = self.basicHeaderObj.utc
1049 1049 return 1
1050 1050 #if self.server is None:
1051 1051 if not(self.setNextFile()):
1052 1052 return 0
1053 1053
1054 1054 deltaTime = self.basicHeaderObj.utc - self.lastUTTime #
1055 1055 self.lastUTTime = self.basicHeaderObj.utc
1056 1056
1057 1057 self.flagDiscontinuousBlock = 0
1058 1058
1059 1059 if deltaTime > self.maxTimeStep:
1060 1060 self.flagDiscontinuousBlock = 1
1061 1061
1062 1062 return 1
1063 1063
1064 1064 def readNextBlock(self):
1065 1065
1066 1066 #Skip block out of startTime and endTime
1067 1067 while True:
1068 1068 if not(self.__setNewBlock()):
1069 1069 print 'returning'
1070 1070 return 0
1071 1071 if not(self.readBlock()):
1072 1072 return 0
1073 1073 self.getBasicHeader()
1074 1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075 1075
1076 1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1077 1077 self.processingHeaderObj.dataBlocksPerFile,
1078 1078 self.dataOut.datatime.ctime())
1079 1079 continue
1080 1080
1081 1081 break
1082 1082
1083 1083 if self.verbose:
1084 1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 1085 self.processingHeaderObj.dataBlocksPerFile,
1086 1086 self.dataOut.datatime.ctime())
1087 1087 return 1
1088 1088
1089 1089 def __readFirstHeader(self):
1090 1090
1091 1091 self.basicHeaderObj.read(self.fp)
1092 1092 self.systemHeaderObj.read(self.fp)
1093 1093 self.radarControllerHeaderObj.read(self.fp)
1094 1094 self.processingHeaderObj.read(self.fp)
1095 1095
1096 1096 self.firstHeaderSize = self.basicHeaderObj.size
1097 1097
1098 1098 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
1099 1099 if datatype == 0:
1100 1100 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
1101 1101 elif datatype == 1:
1102 1102 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
1103 1103 elif datatype == 2:
1104 1104 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
1105 1105 elif datatype == 3:
1106 1106 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
1107 1107 elif datatype == 4:
1108 1108 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
1109 1109 elif datatype == 5:
1110 1110 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
1111 1111 else:
1112 1112 raise ValueError, 'Data type was not defined'
1113 1113
1114 1114 self.dtype = datatype_str
1115 1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 1119 self.getBlockDimension()
1120 1120
1121 1121 def __verifyFile(self, filename, msgFlag=True):
1122 1122
1123 1123 msg = None
1124 1124
1125 1125 try:
1126 1126 fp = open(filename, 'rb')
1127 1127 except IOError:
1128 1128
1129 1129 if msgFlag:
1130 1130 print "[Reading] File %s can't be opened" % (filename)
1131 1131
1132 1132 return False
1133 1133
1134 1134 currentPosition = fp.tell()
1135 1135 neededSize = self.processingHeaderObj.blockSize + self.firstHeaderSize
1136 1136
1137 1137 if neededSize == 0:
1138 1138 basicHeaderObj = BasicHeader(LOCALTIME)
1139 1139 systemHeaderObj = SystemHeader()
1140 1140 radarControllerHeaderObj = RadarControllerHeader()
1141 1141 processingHeaderObj = ProcessingHeader()
1142 1142
1143 1143 if not( basicHeaderObj.read(fp) ):
1144 1144 fp.close()
1145 1145 return False
1146 1146
1147 1147 if not( systemHeaderObj.read(fp) ):
1148 1148 fp.close()
1149 1149 return False
1150 1150
1151 1151 if not( radarControllerHeaderObj.read(fp) ):
1152 1152 fp.close()
1153 1153 return False
1154 1154
1155 1155 if not( processingHeaderObj.read(fp) ):
1156 1156 fp.close()
1157 1157 return False
1158 1158
1159 1159 neededSize = processingHeaderObj.blockSize + basicHeaderObj.size
1160 1160 else:
1161 1161 msg = "[Reading] Skipping the file %s due to it hasn't enough data" %filename
1162 1162
1163 1163 fp.close()
1164 1164
1165 1165 fileSize = os.path.getsize(filename)
1166 1166 currentSize = fileSize - currentPosition
1167 1167
1168 1168 if currentSize < neededSize:
1169 1169 if msgFlag and (msg != None):
1170 1170 print msg
1171 1171 return False
1172 1172
1173 1173 return True
1174 1174
1175 1175 def findDatafiles(self, path, startDate=None, endDate=None, expLabel='', ext='.r', walk=True, include_path=False):
1176 1176
1177 1177 path_empty = True
1178 1178
1179 1179 dateList = []
1180 1180 pathList = []
1181 1181
1182 1182 multi_path = path.split(',')
1183 1183
1184 1184 if not walk:
1185 1185
1186 1186 for single_path in multi_path:
1187 1187
1188 1188 if not os.path.isdir(single_path):
1189 1189 continue
1190 1190
1191 1191 fileList = glob.glob1(single_path, "*"+ext)
1192 1192
1193 1193 if not fileList:
1194 1194 continue
1195 1195
1196 1196 path_empty = False
1197 1197
1198 1198 fileList.sort()
1199 1199
1200 1200 for thisFile in fileList:
1201 1201
1202 1202 if not os.path.isfile(os.path.join(single_path, thisFile)):
1203 1203 continue
1204 1204
1205 1205 if not isRadarFile(thisFile):
1206 1206 continue
1207 1207
1208 1208 if not isFileInDateRange(thisFile, startDate, endDate):
1209 1209 continue
1210 1210
1211 1211 thisDate = getDateFromRadarFile(thisFile)
1212 1212
1213 1213 if thisDate in dateList:
1214 1214 continue
1215 1215
1216 1216 dateList.append(thisDate)
1217 1217 pathList.append(single_path)
1218 1218
1219 1219 else:
1220 1220 for single_path in multi_path:
1221 1221
1222 1222 if not os.path.isdir(single_path):
1223 1223 continue
1224 1224
1225 1225 dirList = []
1226 1226
1227 1227 for thisPath in os.listdir(single_path):
1228 1228
1229 1229 if not os.path.isdir(os.path.join(single_path,thisPath)):
1230 1230 continue
1231 1231
1232 1232 if not isRadarFolder(thisPath):
1233 1233 continue
1234 1234
1235 1235 if not isFolderInDateRange(thisPath, startDate, endDate):
1236 1236 continue
1237 1237
1238 1238 dirList.append(thisPath)
1239 1239
1240 1240 if not dirList:
1241 1241 continue
1242 1242
1243 1243 dirList.sort()
1244 1244
1245 1245 for thisDir in dirList:
1246 1246
1247 1247 datapath = os.path.join(single_path, thisDir, expLabel)
1248 1248 fileList = glob.glob1(datapath, "*"+ext)
1249 1249
1250 1250 if not fileList:
1251 1251 continue
1252 1252
1253 1253 path_empty = False
1254 1254
1255 1255 thisDate = getDateFromRadarFolder(thisDir)
1256 1256
1257 1257 pathList.append(datapath)
1258 1258 dateList.append(thisDate)
1259 1259
1260 1260 dateList.sort()
1261 1261
1262 1262 if walk:
1263 1263 pattern_path = os.path.join(multi_path[0], "[dYYYYDDD]", expLabel)
1264 1264 else:
1265 1265 pattern_path = multi_path[0]
1266 1266
1267 1267 if path_empty:
1268 1268 print "[Reading] No *%s files in %s for %s to %s" %(ext, pattern_path, startDate, endDate)
1269 1269 else:
1270 1270 if not dateList:
1271 1271 print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
1272 1272
1273 1273 if include_path:
1274 1274 return dateList, pathList
1275 1275
1276 1276 return dateList
1277 1277
1278 1278 def setup(self,
1279 1279 path=None,
1280 1280 startDate=None,
1281 1281 endDate=None,
1282 1282 startTime=datetime.time(0,0,0),
1283 1283 endTime=datetime.time(23,59,59),
1284 1284 set=None,
1285 1285 expLabel = "",
1286 1286 ext = None,
1287 1287 online = False,
1288 1288 delay = 60,
1289 1289 walk = True,
1290 1290 getblock = False,
1291 1291 nTxs = 1,
1292 1292 realtime=False,
1293 1293 blocksize=None,
1294 1294 blocktime=None,
1295 1295 verbose=True,
1296 1296 **kwargs):
1297
1297
1298 1298 if path == None:
1299 1299 raise ValueError, "[Reading] The path is not valid"
1300 1300
1301 1301
1302 1302 if ext == None:
1303 1303 ext = self.ext
1304 1304
1305 1305 self.verbose=verbose
1306 1306 self.path = path
1307 1307 self.startDate = startDate
1308 1308 self.endDate = endDate
1309 1309 self.startTime = startTime
1310 1310 self.endTime = endTime
1311 1311 self.set = set
1312 1312 self.expLabel = expLabel
1313 1313 self.ext = ext
1314 1314 self.online = online
1315 1315 self.delay = delay
1316 1316 self.walk = walk
1317 1317 self.getblock = getblock
1318 1318 self.nTxs = nTxs
1319 1319 self.realtime = realtime
1320 1320 self.blocksize = blocksize
1321 1321 self.blocktime = blocktime
1322 1322
1323 1323
1324 1324 if self.firstTime is True:
1325 1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 1326 startTime=startTime, endTime=endTime,
1327 1327 set=set, expLabel=expLabel, ext=ext,
1328 1328 walk=walk)
1329 1329 if filenameList is not None: filenameList = filenameList[:-1]
1330 1330
1331 1331 if pathList is not None and filenameList is not None and online:
1332 1332 self.onlineWithDate = True
1333 1333 online = False
1334 1334 self.fileIndex = -1
1335 1335 self.pathList = pathList
1336 1336 self.filenameList = filenameList
1337 1337 file_name = os.path.basename(filenameList[-1])
1338 1338 basename, ext = os.path.splitext(file_name)
1339 1339 last_set = int(basename[-3:])
1340 1340
1341 1341 if online:
1342 1342 print "[Reading] Searching files in online mode..."
1343 1343
1344 1344 for nTries in range(self.nTries):
1345 1345 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1346 1346 expLabel=expLabel,
1347 1347 ext=ext,
1348 1348 walk=walk,
1349 1349 startDate=startDate,
1350 1350 startTime=startTime,
1351 1351 set=set)
1352 1352
1353 1353 if fullpath:
1354 1354 break
1355 1355 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1356 1356 sleep( self.delay )
1357 1357
1358 1358 if not(fullpath):
1359 1359 print "[Reading] There 'isn't any valid file in %s" % path
1360 1360 return
1361 1361
1362 1362 self.year = year
1363 1363 self.doy = doy
1364 1364 self.set = set - 1
1365 1365 self.path = path
1366 1366 self.foldercounter = foldercounter
1367 1367 last_set = None
1368 1368 else:
1369 1369 print "[Reading] Searching files in offline mode ..."
1370 1370 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1371 1371 startTime=startTime, endTime=endTime,
1372 1372 set=set, expLabel=expLabel, ext=ext,
1373 1373 walk=walk)
1374 1374
1375 1375 if not(pathList):
1376 1376 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1377 1377 # datetime.datetime.combine(startDate,startTime).ctime(),
1378 1378 # datetime.datetime.combine(endDate,endTime).ctime())
1379 1379
1380 1380 # sys.exit(-1)
1381 1381
1382 1382 self.fileIndex = -1
1383 1383 self.pathList = []
1384 1384 self.filenameList = []
1385 1385 return
1386 1386
1387 1387 self.fileIndex = -1
1388 1388 self.pathList = pathList
1389 1389 self.filenameList = filenameList
1390 1390 file_name = os.path.basename(filenameList[-1])
1391 1391 basename, ext = os.path.splitext(file_name)
1392 1392 last_set = int(basename[-3:])
1393 1393
1394 1394
1395 1395 self.online = online
1396 1396 self.realtime = realtime
1397 1397 self.delay = delay
1398 1398 ext = ext.lower()
1399 1399 self.ext = ext
1400 1400 self.getByBlock = getblock
1401 1401 self.nTxs = nTxs
1402 1402 self.startTime = startTime
1403 1403 self.endTime = endTime
1404 1404
1405 1405
1406 1406 #Added-----------------
1407 1407 self.selBlocksize = blocksize
1408 1408 self.selBlocktime = blocktime
1409 1409
1410 1410
1411 1411 if not(self.setNextFile()):
1412 1412 if (startDate!=None) and (endDate!=None):
1413 1413 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1414 1414 elif startDate != None:
1415 1415 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1416 1416 else:
1417 1417 print "[Reading] No files"
1418 1418
1419 1419 self.fileIndex = -1
1420 1420 self.pathList = []
1421 1421 self.filenameList = []
1422 1422 return
1423 1423
1424 1424 # self.getBasicHeader()
1425 1425
1426 1426 if last_set != None:
1427 1427 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1428 1428 return
1429 1429
1430 1430 def getBasicHeader(self):
1431 1431
1432 1432 self.dataOut.utctime = self.basicHeaderObj.utc + self.basicHeaderObj.miliSecond/1000. + self.profileIndex * self.radarControllerHeaderObj.ippSeconds
1433 1433
1434 1434 self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
1435 1435
1436 1436 self.dataOut.timeZone = self.basicHeaderObj.timeZone
1437 1437
1438 1438 self.dataOut.dstFlag = self.basicHeaderObj.dstFlag
1439 1439
1440 1440 self.dataOut.errorCount = self.basicHeaderObj.errorCount
1441 1441
1442 1442 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1443 1443
1444 1444 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1445 1445
1446 1446 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1447 1447
1448 1448
1449 1449 def getFirstHeader(self):
1450 1450
1451 1451 raise NotImplementedError
1452 1452
1453 1453 def getData(self):
1454 1454
1455 1455 raise NotImplementedError
1456 1456
1457 1457 def hasNotDataInBuffer(self):
1458 1458
1459 1459 raise NotImplementedError
1460 1460
1461 1461 def readBlock(self):
1462 1462
1463 1463 raise NotImplementedError
1464 1464
1465 1465 def isEndProcess(self):
1466 1466
1467 1467 return self.flagNoMoreFiles
1468 1468
1469 1469 def printReadBlocks(self):
1470 1470
1471 1471 print "[Reading] Number of read blocks per file %04d" %self.nReadBlocks
1472 1472
1473 1473 def printTotalBlocks(self):
1474 1474
1475 1475 print "[Reading] Number of read blocks %04d" %self.nTotalBlocks
1476 1476
1477 1477 def printNumberOfBlock(self):
1478 1478
1479 1479 if self.flagIsNewBlock:
1480 1480 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1481 1481 self.processingHeaderObj.dataBlocksPerFile,
1482 1482 self.dataOut.datatime.ctime())
1483 1483
1484 1484 def printInfo(self):
1485 1485
1486 1486 if self.__printInfo == False:
1487 1487 return
1488 1488
1489 1489 self.basicHeaderObj.printInfo()
1490 1490 self.systemHeaderObj.printInfo()
1491 1491 self.radarControllerHeaderObj.printInfo()
1492 1492 self.processingHeaderObj.printInfo()
1493 1493
1494 1494 self.__printInfo = False
1495 1495
1496 1496 def run(self,
1497 1497 path=None,
1498 1498 startDate=None,
1499 1499 endDate=None,
1500 1500 startTime=datetime.time(0,0,0),
1501 1501 endTime=datetime.time(23,59,59),
1502 1502 set=None,
1503 1503 expLabel = "",
1504 1504 ext = None,
1505 1505 online = False,
1506 1506 delay = 60,
1507 1507 walk = True,
1508 1508 getblock = False,
1509 1509 nTxs = 1,
1510 1510 realtime=False,
1511 1511 blocksize=None,
1512 1512 blocktime=None,
1513 1513 queue=None,
1514 1514 skip=None,
1515 1515 cursor=None,
1516 1516 warnings=True,
1517 1517 server=None,
1518 1518 verbose=True, **kwargs):
1519
1520 1519 if not(self.isConfig):
1521 1520 # self.dataOut = dataOut
1522 1521 self.setup( path=path,
1523 1522 startDate=startDate,
1524 1523 endDate=endDate,
1525 1524 startTime=startTime,
1526 1525 endTime=endTime,
1527 1526 set=set,
1528 1527 expLabel=expLabel,
1529 1528 ext=ext,
1530 1529 online=online,
1531 1530 delay=delay,
1532 1531 walk=walk,
1533 1532 getblock=getblock,
1534 1533 nTxs=nTxs,
1535 1534 realtime=realtime,
1536 1535 blocksize=blocksize,
1537 1536 blocktime=blocktime,
1538 1537 queue=queue,
1539 1538 skip=skip,
1540 1539 cursor=cursor,
1541 1540 warnings=warnings,
1542 1541 server=server,
1543 1542 verbose=verbose, **kwargs)
1544 1543 self.isConfig = True
1545 1544 if server is None:
1546 1545 self.getData()
1547 1546 else:
1548 1547 self.getFromServer()
1549 1548
1550 1549 class JRODataWriter(JRODataIO):
1551 1550
1552 1551 """
1553 1552 Esta clase permite escribir datos a archivos procesados (.r o ,pdata). La escritura
1554 1553 de los datos siempre se realiza por bloques.
1555 1554 """
1556 1555
1557 1556 blockIndex = 0
1558 1557
1559 1558 path = None
1560 1559
1561 1560 setFile = None
1562 1561
1563 1562 profilesPerBlock = None
1564 1563
1565 1564 blocksPerFile = None
1566 1565
1567 1566 nWriteBlocks = 0
1568 1567
1569 1568 fileDate = None
1570 1569
1571 1570 def __init__(self, dataOut=None):
1572 1571 raise NotImplementedError
1573 1572
1574 1573
1575 1574 def hasAllDataInBuffer(self):
1576 1575 raise NotImplementedError
1577 1576
1578 1577
1579 1578 def setBlockDimension(self):
1580 1579 raise NotImplementedError
1581 1580
1582 1581
1583 1582 def writeBlock(self):
1584 1583 raise NotImplementedError
1585 1584
1586 1585
1587 1586 def putData(self):
1588 1587 raise NotImplementedError
1589 1588
1590 1589
1591 1590 def getProcessFlags(self):
1592 1591
1593 1592 processFlags = 0
1594 1593
1595 1594 dtype_index = get_dtype_index(self.dtype)
1596 1595 procflag_dtype = get_procflag_dtype(dtype_index)
1597 1596
1598 1597 processFlags += procflag_dtype
1599 1598
1600 1599 if self.dataOut.flagDecodeData:
1601 1600 processFlags += PROCFLAG.DECODE_DATA
1602 1601
1603 1602 if self.dataOut.flagDeflipData:
1604 1603 processFlags += PROCFLAG.DEFLIP_DATA
1605 1604
1606 1605 if self.dataOut.code is not None:
1607 1606 processFlags += PROCFLAG.DEFINE_PROCESS_CODE
1608 1607
1609 1608 if self.dataOut.nCohInt > 1:
1610 1609 processFlags += PROCFLAG.COHERENT_INTEGRATION
1611 1610
1612 1611 if self.dataOut.type == "Spectra":
1613 1612 if self.dataOut.nIncohInt > 1:
1614 1613 processFlags += PROCFLAG.INCOHERENT_INTEGRATION
1615 1614
1616 1615 if self.dataOut.data_dc is not None:
1617 1616 processFlags += PROCFLAG.SAVE_CHANNELS_DC
1618 1617
1619 1618 if self.dataOut.flagShiftFFT:
1620 1619 processFlags += PROCFLAG.SHIFT_FFT_DATA
1621 1620
1622 1621 return processFlags
1623 1622
1624 1623 def setBasicHeader(self):
1625 1624
1626 1625 self.basicHeaderObj.size = self.basicHeaderSize #bytes
1627 1626 self.basicHeaderObj.version = self.versionFile
1628 1627 self.basicHeaderObj.dataBlock = self.nTotalBlocks
1629 1628
1630 1629 utc = numpy.floor(self.dataOut.utctime)
1631 1630 milisecond = (self.dataOut.utctime - utc)* 1000.0
1632 1631
1633 1632 self.basicHeaderObj.utc = utc
1634 1633 self.basicHeaderObj.miliSecond = milisecond
1635 1634 self.basicHeaderObj.timeZone = self.dataOut.timeZone
1636 1635 self.basicHeaderObj.dstFlag = self.dataOut.dstFlag
1637 1636 self.basicHeaderObj.errorCount = self.dataOut.errorCount
1638 1637
1639 1638 def setFirstHeader(self):
1640 1639 """
1641 1640 Obtiene una copia del First Header
1642 1641
1643 1642 Affected:
1644 1643
1645 1644 self.basicHeaderObj
1646 1645 self.systemHeaderObj
1647 1646 self.radarControllerHeaderObj
1648 1647 self.processingHeaderObj self.
1649 1648
1650 1649 Return:
1651 1650 None
1652 1651 """
1653 1652
1654 1653 raise NotImplementedError
1655 1654
1656 1655 def __writeFirstHeader(self):
1657 1656 """
1658 1657 Escribe el primer header del file es decir el Basic header y el Long header (SystemHeader, RadarControllerHeader, ProcessingHeader)
1659 1658
1660 1659 Affected:
1661 1660 __dataType
1662 1661
1663 1662 Return:
1664 1663 None
1665 1664 """
1666 1665
1667 1666 # CALCULAR PARAMETROS
1668 1667
1669 1668 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1670 1669 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1671 1670
1672 1671 self.basicHeaderObj.write(self.fp)
1673 1672 self.systemHeaderObj.write(self.fp)
1674 1673 self.radarControllerHeaderObj.write(self.fp)
1675 1674 self.processingHeaderObj.write(self.fp)
1676 1675
1677 1676 def __setNewBlock(self):
1678 1677 """
1679 1678 Si es un nuevo file escribe el First Header caso contrario escribe solo el Basic Header
1680 1679
1681 1680 Return:
1682 1681 0 : si no pudo escribir nada
1683 1682 1 : Si escribio el Basic el First Header
1684 1683 """
1685 1684 if self.fp == None:
1686 1685 self.setNextFile()
1687 1686
1688 1687 if self.flagIsNewFile:
1689 1688 return 1
1690 1689
1691 1690 if self.blockIndex < self.processingHeaderObj.dataBlocksPerFile:
1692 1691 self.basicHeaderObj.write(self.fp)
1693 1692 return 1
1694 1693
1695 1694 if not( self.setNextFile() ):
1696 1695 return 0
1697 1696
1698 1697 return 1
1699 1698
1700 1699
1701 1700 def writeNextBlock(self):
1702 1701 """
1703 1702 Selecciona el bloque siguiente de datos y los escribe en un file
1704 1703
1705 1704 Return:
1706 1705 0 : Si no hizo pudo escribir el bloque de datos
1707 1706 1 : Si no pudo escribir el bloque de datos
1708 1707 """
1709 1708 if not( self.__setNewBlock() ):
1710 1709 return 0
1711 1710
1712 1711 self.writeBlock()
1713 1712
1714 1713 print "[Writing] Block No. %d/%d" %(self.blockIndex,
1715 1714 self.processingHeaderObj.dataBlocksPerFile)
1716 1715
1717 1716 return 1
1718 1717
1719 1718 def setNextFile(self):
1720 1719 """
1721 1720 Determina el siguiente file que sera escrito
1722 1721
1723 1722 Affected:
1724 1723 self.filename
1725 1724 self.subfolder
1726 1725 self.fp
1727 1726 self.setFile
1728 1727 self.flagIsNewFile
1729 1728
1730 1729 Return:
1731 1730 0 : Si el archivo no puede ser escrito
1732 1731 1 : Si el archivo esta listo para ser escrito
1733 1732 """
1734 1733 ext = self.ext
1735 1734 path = self.path
1736 1735
1737 1736 if self.fp != None:
1738 1737 self.fp.close()
1739 1738
1740 1739 timeTuple = time.localtime( self.dataOut.utctime)
1741 1740 subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
1742 1741
1743 1742 fullpath = os.path.join( path, subfolder )
1744 1743 setFile = self.setFile
1745 1744
1746 1745 if not( os.path.exists(fullpath) ):
1747 1746 os.mkdir(fullpath)
1748 1747 setFile = -1 #inicializo mi contador de seteo
1749 1748 else:
1750 1749 filesList = os.listdir( fullpath )
1751 1750 if len( filesList ) > 0:
1752 1751 filesList = sorted( filesList, key=str.lower )
1753 1752 filen = filesList[-1]
1754 1753 # el filename debera tener el siguiente formato
1755 1754 # 0 1234 567 89A BCDE (hex)
1756 1755 # x YYYY DDD SSS .ext
1757 1756 if isNumber( filen[8:11] ):
1758 1757 setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
1759 1758 else:
1760 1759 setFile = -1
1761 1760 else:
1762 1761 setFile = -1 #inicializo mi contador de seteo
1763 1762
1764 1763 setFile += 1
1765 1764
1766 1765 #If this is a new day it resets some values
1767 1766 if self.dataOut.datatime.date() > self.fileDate:
1768 1767 setFile = 0
1769 1768 self.nTotalBlocks = 0
1770 1769
1771 1770 filen = '%s%4.4d%3.3d%3.3d%s' % (self.optchar, timeTuple.tm_year, timeTuple.tm_yday, setFile, ext )
1772 1771
1773 1772 filename = os.path.join( path, subfolder, filen )
1774 1773
1775 1774 fp = open( filename,'wb' )
1776 1775
1777 1776 self.blockIndex = 0
1778 1777
1779 1778 #guardando atributos
1780 1779 self.filename = filename
1781 1780 self.subfolder = subfolder
1782 1781 self.fp = fp
1783 1782 self.setFile = setFile
1784 1783 self.flagIsNewFile = 1
1785 1784 self.fileDate = self.dataOut.datatime.date()
1786 1785
1787 1786 self.setFirstHeader()
1788 1787
1789 1788 print '[Writing] Opening file: %s'%self.filename
1790 1789
1791 1790 self.__writeFirstHeader()
1792 1791
1793 1792 return 1
1794 1793
1795 1794 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True):
1796 1795 """
1797 1796 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1798 1797
1799 1798 Inputs:
1800 1799 path : directory where data will be saved
1801 1800 profilesPerBlock : number of profiles per block
1802 1801 set : initial file set
1803 1802 datatype : An integer number that defines data type:
1804 1803 0 : int8 (1 byte)
1805 1804 1 : int16 (2 bytes)
1806 1805 2 : int32 (4 bytes)
1807 1806 3 : int64 (8 bytes)
1808 1807 4 : float32 (4 bytes)
1809 1808 5 : double64 (8 bytes)
1810 1809
1811 1810 Return:
1812 1811 0 : Si no realizo un buen seteo
1813 1812 1 : Si realizo un buen seteo
1814 1813 """
1815 1814
1816 1815 if ext == None:
1817 1816 ext = self.ext
1818 1817
1819 1818 self.ext = ext.lower()
1820 1819
1821 1820 self.path = path
1822 1821
1823 1822 if set is None:
1824 1823 self.setFile = -1
1825 1824 else:
1826 1825 self.setFile = set - 1
1827 1826
1828 1827 self.blocksPerFile = blocksPerFile
1829 1828
1830 1829 self.profilesPerBlock = profilesPerBlock
1831 1830
1832 1831 self.dataOut = dataOut
1833 1832 self.fileDate = self.dataOut.datatime.date()
1834 1833 #By default
1835 1834 self.dtype = self.dataOut.dtype
1836 1835
1837 1836 if datatype is not None:
1838 1837 self.dtype = get_numpy_dtype(datatype)
1839 1838
1840 1839 if not(self.setNextFile()):
1841 1840 print "[Writing] There isn't a next file"
1842 1841 return 0
1843 1842
1844 1843 self.setBlockDimension()
1845 1844
1846 1845 return 1
1847 1846
1848 1847 def run(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, **kwargs):
1849 1848
1850 1849 if not(self.isConfig):
1851 1850
1852 1851 self.setup(dataOut, path, blocksPerFile, profilesPerBlock=profilesPerBlock, set=set, ext=ext, datatype=datatype, **kwargs)
1853 1852 self.isConfig = True
1854 1853
1855 1854 self.putData()
General Comments 0
You need to be logged in to leave comments. Login now