##// END OF EJS Templates
schain
RSS
formatting
José Chávez -
r1082:831a23834344
parent child
Show More
Side by Side Unified
Context file:
r1082:831a23834344 Loading diff...:
Show file before | Show file after | Hide comments
@@ -24,6 +24,7 DTYPES = {
24 'Spectra': '.pdata'
24 'Spectra': '.pdata'
25 }
25 }
26
26
27
27 def MPProject(project, n=cpu_count()):
28 def MPProject(project, n=cpu_count()):
28 '''
29 '''
29 Project wrapper to run schain in n processes
30 Project wrapper to run schain in n processes
@@ -34,8 +35,8 def MPProject(project, n=cpu_count()):
34 dt1 = op.getParameterValue('startDate')
35 dt1 = op.getParameterValue('startDate')
35 dt2 = op.getParameterValue('endDate')
36 dt2 = op.getParameterValue('endDate')
36 days = (dt2 - dt1).days
37 days = (dt2 - dt1).days
37
38
38 for day in range(days+1):
39 for day in range(days + 1):
39 skip = 0
40 skip = 0
40 cursor = 0
41 cursor = 0
41 processes = []
42 processes = []
@@ -43,17 +44,17 def MPProject(project, n=cpu_count()):
43 dt_str = dt.strftime('%Y/%m/%d')
44 dt_str = dt.strftime('%Y/%m/%d')
44 reader = JRODataReader()
45 reader = JRODataReader()
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 startDate=dt,
47 startDate=dt,
47 endDate=dt,
48 endDate=dt,
48 ext=DTYPES[rconf.datatype])
49 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
50 nFiles = len(files)
50 if nFiles == 0:
51 if nFiles == 0:
51 continue
52 continue
52 skip = int(math.ceil(nFiles/n))
53 skip = int(math.ceil(nFiles / n))
53 while nFiles > cursor*skip:
54 while nFiles > cursor * skip:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 skip=skip)
56 skip=skip)
56 p = project.clone()
57 p = project.clone()
57 p.start()
58 p.start()
58 processes.append(p)
59 processes.append(p)
59 cursor += 1
60 cursor += 1
@@ -72,6 +73,7 def MPProject(project, n=cpu_count()):
72
73
73 time.sleep(3)
74 time.sleep(3)
74
75
76
75 class ParameterConf():
77 class ParameterConf():
76
78
77 id = None
79 id = None
@@ -108,7 +110,7 class ParameterConf():
108 return self.__formated_value
110 return self.__formated_value
109
111
110 if value == '':
112 if value == '':
111 raise ValueError, '%s: This parameter value is empty' %self.name
113 raise ValueError, '%s: This parameter value is empty' % self.name
112
114
113 if format == 'list':
115 if format == 'list':
114 strList = value.split(',')
116 strList = value.split(',')
@@ -174,16 +176,16 class ParameterConf():
174 new_value = ast.literal_eval(value)
176 new_value = ast.literal_eval(value)
175
177
176 if type(new_value) not in (tuple, list):
178 if type(new_value) not in (tuple, list):
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
179 raise ValueError, '%s has to be a tuple or list of pairs' % value
178
180
179 if type(new_value[0]) not in (tuple, list):
181 if type(new_value[0]) not in (tuple, list):
180 if len(new_value) != 2:
182 if len(new_value) != 2:
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
183 raise ValueError, '%s has to be a tuple or list of pairs' % value
182 new_value = [new_value]
184 new_value = [new_value]
183
185
184 for thisPair in new_value:
186 for thisPair in new_value:
185 if len(thisPair) != 2:
187 if len(thisPair) != 2:
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
188 raise ValueError, '%s has to be a tuple or list of pairs' % value
187
189
188 self.__formated_value = new_value
190 self.__formated_value = new_value
189
191
@@ -253,13 +255,14 class ParameterConf():
253 self.value = parmElement.get('value')
255 self.value = parmElement.get('value')
254 self.format = str.lower(parmElement.get('format'))
256 self.format = str.lower(parmElement.get('format'))
255
257
256 #Compatible with old signal chain version
258 # Compatible with old signal chain version
257 if self.format == 'int' and self.name == 'idfigure':
259 if self.format == 'int' and self.name == 'idfigure':
258 self.name = 'id'
260 self.name = 'id'
259
261
260 def printattr(self):
262 def printattr(self):
261
263
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
264 print 'Parameter[%s]: name = %s, value = %s, format = %s' % (self.id, self.name, self.value, self.format)
265
263
266
264 class OperationConf():
267 class OperationConf():
265
268
@@ -279,10 +282,9 class OperationConf():
279 self.priority = None
282 self.priority = None
280 self.type = 'self'
283 self.type = 'self'
281
284
282
283 def __getNewId(self):
285 def __getNewId(self):
284
286
285 return int(self.id)*10 + len(self.parmConfObjList) + 1
287 return int(self.id) * 10 + len(self.parmConfObjList) + 1
286
288
287 def updateId(self, new_id):
289 def updateId(self, new_id):
288
290
@@ -291,7 +293,7 class OperationConf():
291 n = 1
293 n = 1
292 for parmObj in self.parmConfObjList:
294 for parmObj in self.parmConfObjList:
293
295
294 idParm = str(int(new_id)*10 + n)
296 idParm = str(int(new_id) * 10 + n)
295 parmObj.updateId(idParm)
297 parmObj.updateId(idParm)
296
298
297 n += 1
299 n += 1
@@ -329,15 +331,14 class OperationConf():
329 def getParameterValue(self, parameterName):
331 def getParameterValue(self, parameterName):
330
332
331 parameterObj = self.getParameterObj(parameterName)
333 parameterObj = self.getParameterObj(parameterName)
332
334
333 # if not parameterObj:
335 # if not parameterObj:
334 # return None
336 # return None
335
337
336 value = parameterObj.getValue()
338 value = parameterObj.getValue()
337
339
338 return value
340 return value
339
341
340
341 def getKwargs(self):
342 def getKwargs(self):
342
343
343 kwargs = {}
344 kwargs = {}
@@ -367,7 +368,7 class OperationConf():
367 self.parmConfObjList = []
368 self.parmConfObjList = []
368
369
369 def addParameter(self, name, value, format='str'):
370 def addParameter(self, name, value, format='str'):
370
371
371 if value is None:
372 if value is None:
372 return None
373 return None
373 id = self.__getNewId()
374 id = self.__getNewId()
@@ -405,8 +406,8 class OperationConf():
405 self.type = opElement.get('type')
406 self.type = opElement.get('type')
406 self.priority = opElement.get('priority')
407 self.priority = opElement.get('priority')
407
408
408 #Compatible with old signal chain version
409 # Compatible with old signal chain version
409 #Use of 'run' method instead 'init'
410 # Use of 'run' method instead 'init'
410 if self.type == 'self' and self.name == 'init':
411 if self.type == 'self' and self.name == 'init':
411 self.name = 'run'
412 self.name = 'run'
412
413
@@ -418,8 +419,8 class OperationConf():
418 parmConfObj = ParameterConf()
419 parmConfObj = ParameterConf()
419 parmConfObj.readXml(parmElement)
420 parmConfObj.readXml(parmElement)
420
421
421 #Compatible with old signal chain version
422 # Compatible with old signal chain version
422 #If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
423 # If an 'plot' OPERATION is found, changes name operation by the value of its type PARAMETER
423 if self.type != 'self' and self.name == 'Plot':
424 if self.type != 'self' and self.name == 'Plot':
424 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
425 if parmConfObj.format == 'str' and parmConfObj.name == 'type':
425 self.name = parmConfObj.value
426 self.name = parmConfObj.value
@@ -429,22 +430,21 class OperationConf():
429
430
430 def printattr(self):
431 def printattr(self):
431
432
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
433 print '%s[%s]: name = %s, type = %s, priority = %s' % (self.ELEMENTNAME,
433 self.id,
434 self.id,
434 self.name,
435 self.name,
435 self.type,
436 self.type,
436 self.priority)
437 self.priority)
437
438
438 for parmConfObj in self.parmConfObjList:
439 for parmConfObj in self.parmConfObjList:
439 parmConfObj.printattr()
440 parmConfObj.printattr()
440
441
441 def createObject(self, plotter_queue=None):
442 def createObject(self, plotter_queue=None):
442
443
443
444 if self.type == 'self':
444 if self.type == 'self':
445 raise ValueError, 'This operation type cannot be created'
445 raise ValueError, 'This operation type cannot be created'
446
446
447 if self.type == 'plotter':
447 if self.type == 'plotter':
448 if not plotter_queue:
448 if not plotter_queue:
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
450
450
@@ -489,11 +489,11 class ProcUnitConf():
489
489
490 def __getPriority(self):
490 def __getPriority(self):
491
491
492 return len(self.opConfObjList)+1
492 return len(self.opConfObjList) + 1
493
493
494 def __getNewId(self):
494 def __getNewId(self):
495
495
496 return int(self.id)*10 + len(self.opConfObjList) + 1
496 return int(self.id) * 10 + len(self.opConfObjList) + 1
497
497
498 def getElementName(self):
498 def getElementName(self):
499
499
@@ -505,18 +505,17 class ProcUnitConf():
505
505
506 def updateId(self, new_id, parentId=parentId):
506 def updateId(self, new_id, parentId=parentId):
507
507
508 new_id = int(parentId) * 10 + (int(self.id) % 10)
509 new_inputId = int(parentId) * 10 + (int(self.inputId) % 10)
508
510
509 new_id = int(parentId)*10 + (int(self.id) % 10)
511 # If this proc unit has not inputs
510 new_inputId = int(parentId)*10 + (int(self.inputId) % 10)
511
512 #If this proc unit has not inputs
513 if self.inputId == '0':
512 if self.inputId == '0':
514 new_inputId = 0
513 new_inputId = 0
515
514
516 n = 1
515 n = 1
517 for opConfObj in self.opConfObjList:
516 for opConfObj in self.opConfObjList:
518
517
519 idOp = str(int(new_id)*10 + n)
518 idOp = str(int(new_id) * 10 + n)
520 opConfObj.updateId(idOp)
519 opConfObj.updateId(idOp)
521
520
522 n += 1
521 n += 1
@@ -525,7 +524,6 class ProcUnitConf():
525 self.id = str(new_id)
524 self.id = str(new_id)
526 self.inputId = str(new_inputId)
525 self.inputId = str(new_inputId)
527
526
528
529 def getInputId(self):
527 def getInputId(self):
530
528
531 return self.inputId
529 return self.inputId
@@ -559,18 +557,18 class ProcUnitConf():
559
557
560 def setup(self, id, name, datatype, inputId, parentId=None):
558 def setup(self, id, name, datatype, inputId, parentId=None):
561
559
562 #Compatible with old signal chain version
560 # Compatible with old signal chain version
563 if datatype==None and name==None:
561 if datatype == None and name == None:
564 raise ValueError, 'datatype or name should be defined'
562 raise ValueError, 'datatype or name should be defined'
565
563
566 if name==None:
564 if name == None:
567 if 'Proc' in datatype:
565 if 'Proc' in datatype:
568 name = datatype
566 name = datatype
569 else:
567 else:
570 name = '%sProc' %(datatype)
568 name = '%sProc' % (datatype)
571
569
572 if datatype==None:
570 if datatype == None:
573 datatype = name.replace('Proc','')
571 datatype = name.replace('Proc', '')
574
572
575 self.id = str(id)
573 self.id = str(id)
576 self.name = name
574 self.name = name
@@ -650,16 +648,15 class ProcUnitConf():
650
648
651 def printattr(self):
649 def printattr(self):
652
650
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
651 print '%s[%s]: name = %s, datatype = %s, inputId = %s' % (self.ELEMENTNAME,
654 self.id,
652 self.id,
655 self.name,
653 self.name,
656 self.datatype,
654 self.datatype,
657 self.inputId)
655 self.inputId)
658
656
659 for opConfObj in self.opConfObjList:
657 for opConfObj in self.opConfObjList:
660 opConfObj.printattr()
658 opConfObj.printattr()
661
659
662
663 def getKwargs(self):
660 def getKwargs(self):
664
661
665 opObj = self.opConfObjList[0]
662 opObj = self.opConfObjList[0]
@@ -675,10 +672,11 class ProcUnitConf():
675
672
676 for opConfObj in self.opConfObjList:
673 for opConfObj in self.opConfObjList:
677
674
678 if opConfObj.type=='self' and self.name=='run':
675 if opConfObj.type == 'self' and self.name == 'run':
679 continue
676 continue
680 elif opConfObj.type=='self':
677 elif opConfObj.type == 'self':
681 procUnitObj.addOperationKwargs(opConfObj.id, **opConfObj.getKwargs())
678 procUnitObj.addOperationKwargs(
679 opConfObj.id, **opConfObj.getKwargs())
682 continue
680 continue
683
681
684 opObj = opConfObj.createObject(plotter_queue)
682 opObj = opConfObj.createObject(plotter_queue)
@@ -704,10 +702,10 class ProcUnitConf():
704
702
705 kwargs[parmConfObj.name] = parmConfObj.getValue()
703 kwargs[parmConfObj.name] = parmConfObj.getValue()
706
704
707 sts = self.procUnitObj.call(opType = opConfObj.type,
705 sts = self.procUnitObj.call(opType=opConfObj.type,
708 opName = opConfObj.name,
706 opName=opConfObj.name,
709 opId = opConfObj.id)
707 opId=opConfObj.id)
710
708
711 is_ok = is_ok or sts
709 is_ok = is_ok or sts
712
710
713 return is_ok
711 return is_ok
@@ -725,6 +723,7 class ProcUnitConf():
725
723
726 return
724 return
727
725
726
728 class ReadUnitConf(ProcUnitConf):
727 class ReadUnitConf(ProcUnitConf):
729
728
730 path = None
729 path = None
@@ -754,17 +753,17 class ReadUnitConf(ProcUnitConf):
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
753 def setup(self, id, name, datatype, path='', startDate='', endDate='',
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
754 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
755
757 #Compatible with old signal chain version
756 # Compatible with old signal chain version
758 if datatype==None and name==None:
757 if datatype == None and name == None:
759 raise ValueError, 'datatype or name should be defined'
758 raise ValueError, 'datatype or name should be defined'
760
759
761 if name==None:
760 if name == None:
762 if 'Reader' in datatype:
761 if 'Reader' in datatype:
763 name = datatype
762 name = datatype
764 else:
763 else:
765 name = '%sReader' %(datatype)
764 name = '%sReader' % (datatype)
766 if datatype==None:
765 if datatype == None:
767 datatype = name.replace('Reader','')
766 datatype = name.replace('Reader', '')
768
767
769 self.id = id
768 self.id = id
770 self.name = name
769 self.name = name
@@ -787,15 +786,16 class ReadUnitConf(ProcUnitConf):
787 if 'Reader' in datatype:
786 if 'Reader' in datatype:
788 self.name = datatype
787 self.name = datatype
789 else:
788 else:
790 self.name = '%sReader' %(datatype)
789 self.name = '%sReader' % (datatype)
791 self.datatype = self.name.replace('Reader', '')
790 self.datatype = self.name.replace('Reader', '')
792
791
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
792 attrs = ('path', 'startDate', 'endDate',
794
793 'startTime', 'endTime', 'parentId')
794
795 for attr in attrs:
795 for attr in attrs:
796 if attr in kwargs:
796 if attr in kwargs:
797 setattr(self, attr, kwargs.pop(attr))
797 setattr(self, attr, kwargs.pop(attr))
798
798
799 self.inputId = '0'
799 self.inputId = '0'
800 self.updateRunOperation(**kwargs)
800 self.updateRunOperation(**kwargs)
801
801
@@ -808,21 +808,26 class ReadUnitConf(ProcUnitConf):
808
808
809 def addRunOperation(self, **kwargs):
809 def addRunOperation(self, **kwargs):
810
810
811 opObj = self.addOperation(name = 'run', optype = 'self')
811 opObj = self.addOperation(name='run', optype='self')
812
812
813 if self.server is None:
813 if self.server is None:
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
814 opObj.addParameter(
815 name='datatype', value=self.datatype, format='str')
815 opObj.addParameter(name='path', value=self.path, format='str')
816 opObj.addParameter(name='path', value=self.path, format='str')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
817 opObj.addParameter(
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
818 name='startDate', value=self.startDate, format='date')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
819 opObj.addParameter(
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
820 name='endDate', value=self.endDate, format='date')
820
821 opObj.addParameter(
822 name='startTime', value=self.startTime, format='time')
823 opObj.addParameter(
824 name='endTime', value=self.endTime, format='time')
825
821 for key, value in kwargs.items():
826 for key, value in kwargs.items():
822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
827 opObj.addParameter(name=key, value=value,
828 format=type(value).__name__)
823 else:
829 else:
824 opObj.addParameter(name='server' , value=self.server, format='str')
830 opObj.addParameter(name='server', value=self.server, format='str')
825
826
831
827 return opObj
832 return opObj
828
833
@@ -833,16 +838,19 class ReadUnitConf(ProcUnitConf):
833
838
834 opObj.addParameter(name='datatype', value=self.datatype, format='str')
839 opObj.addParameter(name='datatype', value=self.datatype, format='str')
835 opObj.addParameter(name='path', value=self.path, format='str')
840 opObj.addParameter(name='path', value=self.path, format='str')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
841 opObj.addParameter(
842 name='startDate', value=self.startDate, format='date')
837 opObj.addParameter(name='endDate', value=self.endDate, format='date')
843 opObj.addParameter(name='endDate', value=self.endDate, format='date')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
844 opObj.addParameter(
845 name='startTime', value=self.startTime, format='time')
839 opObj.addParameter(name='endTime', value=self.endTime, format='time')
846 opObj.addParameter(name='endTime', value=self.endTime, format='time')
840
847
841 for key, value in kwargs.items():
848 for key, value in kwargs.items():
842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
849 opObj.addParameter(name=key, value=value,
850 format=type(value).__name__)
843
851
844 return opObj
852 return opObj
845
853
846 def readXml(self, upElement):
854 def readXml(self, upElement):
847
855
848 self.id = upElement.get('id')
856 self.id = upElement.get('id')
@@ -872,6 +880,7 class ReadUnitConf(ProcUnitConf):
872 self.startTime = opConfObj.getParameterValue('startTime')
880 self.startTime = opConfObj.getParameterValue('startTime')
873 self.endTime = opConfObj.getParameterValue('endTime')
881 self.endTime = opConfObj.getParameterValue('endTime')
874
882
883
875 class Project(Process):
884 class Project(Process):
876
885
877 id = None
886 id = None
@@ -900,7 +909,7 class Project(Process):
900
909
901 idList = self.procUnitConfObjDict.keys()
910 idList = self.procUnitConfObjDict.keys()
902
911
903 id = int(self.id)*10
912 id = int(self.id) * 10
904
913
905 while True:
914 while True:
906 id += 1
915 id += 1
@@ -933,8 +942,8 class Project(Process):
933 for procKey in keyList:
942 for procKey in keyList:
934
943
935 procUnitConfObj = self.procUnitConfObjDict[procKey]
944 procUnitConfObj = self.procUnitConfObjDict[procKey]
936 idProcUnit = str(int(self.id)*10 + n)
945 idProcUnit = str(int(self.id) * 10 + n)
937 procUnitConfObj.updateId(idProcUnit, parentId = self.id)
946 procUnitConfObj.updateId(idProcUnit, parentId=self.id)
938
947
939 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
948 newProcUnitConfObjDict[idProcUnit] = procUnitConfObj
940 n += 1
949 n += 1
@@ -944,9 +953,9 class Project(Process):
944 def setup(self, id, name='', description=''):
953 def setup(self, id, name='', description=''):
945
954
946 print
955 print
947 print '*'*60
956 print '*' * 60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
957 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
958 print '*' * 60
950 print
959 print
951 self.id = str(id)
960 self.id = str(id)
952 self.description = description
961 self.description = description
@@ -969,7 +978,8 class Project(Process):
969 idReadUnit = str(id)
978 idReadUnit = str(id)
970
979
971 readUnitConfObj = ReadUnitConf()
980 readUnitConfObj = ReadUnitConf()
972 readUnitConfObj.setup(idReadUnit, name, datatype, parentId=self.id, **kwargs)
981 readUnitConfObj.setup(idReadUnit, name, datatype,
982 parentId=self.id, **kwargs)
973
983
974 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
984 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
975
985
@@ -980,7 +990,8 class Project(Process):
980 idProcUnit = self.__getNewId()
990 idProcUnit = self.__getNewId()
981
991
982 procUnitConfObj = ProcUnitConf()
992 procUnitConfObj = ProcUnitConf()
983 procUnitConfObj.setup(idProcUnit, name, datatype, inputId, parentId=self.id)
993 procUnitConfObj.setup(idProcUnit, name, datatype,
994 inputId, parentId=self.id)
984
995
985 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
996 self.procUnitConfObjDict[procUnitConfObj.getId()] = procUnitConfObj
986
997
@@ -1054,11 +1065,11 class Project(Process):
1054 abs_file = os.path.abspath(filename)
1065 abs_file = os.path.abspath(filename)
1055
1066
1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1067 if not os.access(os.path.dirname(abs_file), os.W_OK):
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1068 print 'No write permission on %s' % os.path.dirname(abs_file)
1058 return 0
1069 return 0
1059
1070
1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1071 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1072 print 'File %s already exists and it could not be overwriten' % abs_file
1062 return 0
1073 return 0
1063
1074
1064 self.makeXml()
1075 self.makeXml()
@@ -1069,7 +1080,7 class Project(Process):
1069
1080
1070 return 1
1081 return 1
1071
1082
1072 def readXml(self, filename = None):
1083 def readXml(self, filename=None):
1073
1084
1074 if not filename:
1085 if not filename:
1075 print 'filename is not defined'
1086 print 'filename is not defined'
@@ -1078,7 +1089,7 class Project(Process):
1078 abs_file = os.path.abspath(filename)
1089 abs_file = os.path.abspath(filename)
1079
1090
1080 if not os.path.isfile(abs_file):
1091 if not os.path.isfile(abs_file):
1081 print '%s file does not exist' %abs_file
1092 print '%s file does not exist' % abs_file
1082 return 0
1093 return 0
1083
1094
1084 self.projectElement = None
1095 self.projectElement = None
@@ -1087,16 +1098,17 class Project(Process):
1087 try:
1098 try:
1088 self.projectElement = ElementTree().parse(abs_file)
1099 self.projectElement = ElementTree().parse(abs_file)
1089 except:
1100 except:
1090 print 'Error reading %s, verify file format' %filename
1101 print 'Error reading %s, verify file format' % filename
1091 return 0
1102 return 0
1092
1103
1093 self.project = self.projectElement.tag
1104 self.project = self.projectElement.tag
1094
1105
1095 self.id = self.projectElement.get('id')
1106 self.id = self.projectElement.get('id')
1096 self.name = self.projectElement.get('name')
1107 self.name = self.projectElement.get('name')
1097 self.description = self.projectElement.get('description')
1108 self.description = self.projectElement.get('description')
1098
1109
1099 readUnitElementList = self.projectElement.iter(ReadUnitConf().getElementName())
1110 readUnitElementList = self.projectElement.iter(
1111 ReadUnitConf().getElementName())
1100
1112
1101 for readUnitElement in readUnitElementList:
1113 for readUnitElement in readUnitElementList:
1102 readUnitConfObj = ReadUnitConf()
1114 readUnitConfObj = ReadUnitConf()
@@ -1107,7 +1119,8 class Project(Process):
1107
1119
1108 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1120 self.procUnitConfObjDict[readUnitConfObj.getId()] = readUnitConfObj
1109
1121
1110 procUnitElementList = self.projectElement.iter(ProcUnitConf().getElementName())
1122 procUnitElementList = self.projectElement.iter(
1123 ProcUnitConf().getElementName())
1111
1124
1112 for procUnitElement in procUnitElementList:
1125 for procUnitElement in procUnitElementList:
1113 procUnitConfObj = ProcUnitConf()
1126 procUnitConfObj = ProcUnitConf()
@@ -1124,10 +1137,10 class Project(Process):
1124
1137
1125 def printattr(self):
1138 def printattr(self):
1126
1139
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1140 print 'Project[%s]: name = %s, description = %s' % (self.id,
1128 self.name,
1141 self.name,
1129 self.description)
1142 self.description)
1130
1143
1131 for procUnitConfObj in self.procUnitConfObjDict.values():
1144 for procUnitConfObj in self.procUnitConfObjDict.values():
1132 procUnitConfObj.printattr()
1145 procUnitConfObj.printattr()
1133
1146
@@ -1149,11 +1162,11 class Project(Process):
1149 if int(inputId) == 0:
1162 if int(inputId) == 0:
1150 continue
1163 continue
1151
1164
1152 #Get input object
1165 # Get input object
1153 puConfINObj = self.procUnitConfObjDict[inputId]
1166 puConfINObj = self.procUnitConfObjDict[inputId]
1154 puObjIN = puConfINObj.getProcUnitObj()
1167 puObjIN = puConfINObj.getProcUnitObj()
1155
1168
1156 #Get current object
1169 # Get current object
1157 thisPUObj = thisPUConfObj.getProcUnitObj()
1170 thisPUObj = thisPUConfObj.getProcUnitObj()
1158
1171
1159 self.__connect(puObjIN, thisPUObj)
1172 self.__connect(puObjIN, thisPUObj)
@@ -1163,11 +1176,11 class Project(Process):
1163 import socket
1176 import socket
1164
1177
1165 err = traceback.format_exception(sys.exc_info()[0],
1178 err = traceback.format_exception(sys.exc_info()[0],
1166 sys.exc_info()[1],
1179 sys.exc_info()[1],
1167 sys.exc_info()[2])
1180 sys.exc_info()[2])
1168
1181
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1182 print '***** Error occurred in %s *****' % (procUnitConfObj.name)
1170 print '***** %s' %err[-1]
1183 print '***** %s' % err[-1]
1171
1184
1172 message = ''.join(err)
1185 message = ''.join(err)
1173
1186
@@ -1176,30 +1189,33 class Project(Process):
1176 if not send_email:
1189 if not send_email:
1177 return
1190 return
1178
1191
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1192 subject = 'SChain v%s: Error running %s\n' % (
1193 schainpy.__version__, procUnitConfObj.name)
1180
1194
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1195 subtitle = '%s: %s\n' % (
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1196 procUnitConfObj.getElementName(), procUnitConfObj.name)
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1197 subtitle += 'Hostname: %s\n' % socket.gethostbyname(
1184 subtitle += 'Configuration file: %s\n' %self.filename
1198 socket.gethostname())
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1199 subtitle += 'Working directory: %s\n' % os.path.abspath('./')
1200 subtitle += 'Configuration file: %s\n' % self.filename
1201 subtitle += 'Time: %s\n' % str(datetime.datetime.now())
1186
1202
1187 readUnitConfObj = self.getReadUnitObj()
1203 readUnitConfObj = self.getReadUnitObj()
1188 if readUnitConfObj:
1204 if readUnitConfObj:
1189 subtitle += '\nInput parameters:\n'
1205 subtitle += '\nInput parameters:\n'
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1206 subtitle += '[Data path = %s]\n' % readUnitConfObj.path
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1207 subtitle += '[Data type = %s]\n' % readUnitConfObj.datatype
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1208 subtitle += '[Start date = %s]\n' % readUnitConfObj.startDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1209 subtitle += '[End date = %s]\n' % readUnitConfObj.endDate
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1210 subtitle += '[Start time = %s]\n' % readUnitConfObj.startTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1211 subtitle += '[End time = %s]\n' % readUnitConfObj.endTime
1196
1212
1197 adminObj = schainpy.admin.SchainNotify()
1213 adminObj = schainpy.admin.SchainNotify()
1198 adminObj.sendAlert(message=message,
1214 adminObj.sendAlert(message=message,
1199 subject=subject,
1215 subject=subject,
1200 subtitle=subtitle,
1216 subtitle=subtitle,
1201 filename=self.filename)
1217 filename=self.filename)
1202
1218
1203 def isPaused(self):
1219 def isPaused(self):
1204 return 0
1220 return 0
1205
1221
@@ -1250,7 +1266,7 class Project(Process):
1250 def run(self):
1266 def run(self):
1251
1267
1252 log.success('Starting {}'.format(self.name))
1268 log.success('Starting {}'.format(self.name))
1253
1269
1254 self.createObjects()
1270 self.createObjects()
1255 self.connectObjects()
1271 self.connectObjects()
1256
1272
@@ -1282,14 +1298,14 class Project(Process):
1282 is_ok = False
1298 is_ok = False
1283 break
1299 break
1284
1300
1285 #If every process unit finished so end process
1301 # If every process unit finished so end process
1286 if not(is_ok):
1302 if not(is_ok):
1287 break
1303 break
1288
1304
1289 if not self.runController():
1305 if not self.runController():
1290 break
1306 break
1291
1307
1292 #Closing every process
1308 # Closing every process
1293 for procKey in keyList:
1309 for procKey in keyList:
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1310 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 procUnitConfObj.close()
1311 procUnitConfObj.close()
Show file before | Show file after | Hide comments
@@ -15,6 +15,7 import tempfile
15 from StringIO import StringIO
15 from StringIO import StringIO
16 # from _sha import blocksize
16 # from _sha import blocksize
17
17
18
18 class VoltageReader(JRODataReader, ProcessingUnit):
19 class VoltageReader(JRODataReader, ProcessingUnit):
19 """
20 """
20 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
21 Esta clase permite leer datos de voltage desde archivos en formato rawdata (.r). La lectura
@@ -134,13 +135,13 class VoltageReader(JRODataReader, ProcessingUnit):
134
135
135 self.path = None
136 self.path = None
136
137
137 self.profileIndex = 2**32-1
138 self.profileIndex = 2**32 - 1
138
139
139 self.delay = 3 #seconds
140 self.delay = 3 # seconds
140
141
141 self.nTries = 3 #quantity tries
142 self.nTries = 3 # quantity tries
142
143
143 self.nFiles = 3 #number of files for searching
144 self.nFiles = 3 # number of files for searching
144
145
145 self.nReadBlocks = 0
146 self.nReadBlocks = 0
146
147
@@ -172,51 +173,49 class VoltageReader(JRODataReader, ProcessingUnit):
172
173
173 def __hasNotDataInBuffer(self):
174 def __hasNotDataInBuffer(self):
174
175
175 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock*self.nTxs:
176 if self.profileIndex >= self.processingHeaderObj.profilesPerBlock * self.nTxs:
176 return 1
177 return 1
177
178
178 return 0
179 return 0
179
180
180
181 def getBlockDimension(self):
181 def getBlockDimension(self):
182 """
182 """
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
183 Obtiene la cantidad de puntos a leer por cada bloque de datos
184
184
185 Affected:
185 Affected:
186 self.blocksize
186 self.blocksize
187
187
188 Return:
188 Return:
189 None
189 None
190 """
190 """
191 pts2read = self.processingHeaderObj.profilesPerBlock * self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
191 pts2read = self.processingHeaderObj.profilesPerBlock * \
192 self.processingHeaderObj.nHeights * self.systemHeaderObj.nChannels
192 self.blocksize = pts2read
193 self.blocksize = pts2read
193
194
194
195
196 def readBlock(self):
195 def readBlock(self):
197 """
196 """
198 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
197 readBlock lee el bloque de datos desde la posicion actual del puntero del archivo
199 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
198 (self.fp) y actualiza todos los parametros relacionados al bloque de datos
200 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
199 (metadata + data). La data leida es almacenada en el buffer y el contador del buffer
201 es seteado a 0
200 es seteado a 0
202
201
203 Inputs:
202 Inputs:
204 None
203 None
205
204
206 Return:
205 Return:
207 None
206 None
208
207
209 Affected:
208 Affected:
210 self.profileIndex
209 self.profileIndex
211 self.datablock
210 self.datablock
212 self.flagIsNewFile
211 self.flagIsNewFile
213 self.flagIsNewBlock
212 self.flagIsNewBlock
214 self.nTotalBlocks
213 self.nTotalBlocks
215
214
216 Exceptions:
215 Exceptions:
217 Si un bloque leido no es un bloque valido
216 Si un bloque leido no es un bloque valido
218 """
217 """
219
218
220 # if self.server is not None:
219 # if self.server is not None:
221 # self.zBlock = self.receiver.recv()
220 # self.zBlock = self.receiver.recv()
222 # self.zHeader = self.zBlock[:24]
221 # self.zHeader = self.zBlock[:24]
@@ -227,22 +226,24 class VoltageReader(JRODataReader, ProcessingUnit):
227 # self.systemHeaderObj.nChannels
226 # self.systemHeaderObj.nChannels
228 # else:
227 # else:
229 current_pointer_location = self.fp.tell()
228 current_pointer_location = self.fp.tell()
230 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
229 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
231
230
232 try:
231 try:
233 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
232 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
233 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
234 except:
234 except:
235 #print "The read block (%3d) has not enough data" %self.nReadBlocks
235 # print "The read block (%3d) has not enough data" %self.nReadBlocks
236
236
237 if self.waitDataBlock(pointer_location=current_pointer_location):
237 if self.waitDataBlock(pointer_location=current_pointer_location):
238 junk = numpy.fromfile( self.fp, self.dtype, self.blocksize )
238 junk = numpy.fromfile(self.fp, self.dtype, self.blocksize)
239 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
239 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
240 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
240 # return 0
241 # return 0
241
242
242 #Dimensions : nChannels, nProfiles, nSamples
243 # Dimensions : nChannels, nProfiles, nSamples
243
244
244 junk = numpy.transpose(junk, (2,0,1))
245 junk = numpy.transpose(junk, (2, 0, 1))
245 self.datablock = junk['real'] + junk['imag']*1j
246 self.datablock = junk['real'] + junk['imag'] * 1j
246
247
247 self.profileIndex = 0
248 self.profileIndex = 0
248
249
@@ -263,8 +264,8 class VoltageReader(JRODataReader, ProcessingUnit):
263 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
264 self.dataOut.radarControllerHeaderObj = self.radarControllerHeaderObj.copy()
264
265
265 if self.nTxs > 1:
266 if self.nTxs > 1:
266 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
267 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
267 #Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
268 # Time interval and code are propierties of dataOut. Its value depends of radarControllerHeaderObj.
268
269
269 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
270 # self.dataOut.timeInterval = self.radarControllerHeaderObj.ippSeconds * self.processingHeaderObj.nCohInt
270 #
271 #
@@ -280,15 +281,18 class VoltageReader(JRODataReader, ProcessingUnit):
280
281
281 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
282 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock
282
283
283 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
284 self.dataOut.heightList = numpy.arange(
285 self.processingHeaderObj.nHeights) * self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
284
286
285 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
287 self.dataOut.channelList = range(self.systemHeaderObj.nChannels)
286
288
287 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
289 self.dataOut.nCohInt = self.processingHeaderObj.nCohInt
288
290
289 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode #asumo q la data no esta decodificada
291 # asumo q la data no esta decodificada
292 self.dataOut.flagDecodeData = self.processingHeaderObj.flag_decode
290
293
291 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip #asumo q la data no esta sin flip
294 # asumo q la data no esta sin flip
295 self.dataOut.flagDeflipData = self.processingHeaderObj.flag_deflip
292
296
293 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
297 self.dataOut.flagShiftFFT = self.processingHeaderObj.shif_fft
294
298
@@ -300,51 +304,57 class VoltageReader(JRODataReader, ProcessingUnit):
300 if self.nTxs == 1:
304 if self.nTxs == 1:
301 return
305 return
302
306
303 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1./self.nTxs) != 0:
307 if self.nTxs < 1 and self.processingHeaderObj.profilesPerBlock % (1. / self.nTxs) != 0:
304 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" %(1./self.nTxs, self.processingHeaderObj.profilesPerBlock)
308 raise ValueError, "1./nTxs (=%f), should be a multiple of nProfiles (=%d)" % (
309 1. / self.nTxs, self.processingHeaderObj.profilesPerBlock)
305
310
306 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
311 if self.nTxs > 1 and self.processingHeaderObj.nHeights % self.nTxs != 0:
307 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" %(self.nTxs, self.processingHeaderObj.nHeights)
312 raise ValueError, "nTxs (=%d), should be a multiple of nHeights (=%d)" % (
313 self.nTxs, self.processingHeaderObj.nHeights)
308
314
309 self.datablock = self.datablock.reshape((self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock*self.nTxs, self.processingHeaderObj.nHeights/self.nTxs))
315 self.datablock = self.datablock.reshape(
316 (self.systemHeaderObj.nChannels, self.processingHeaderObj.profilesPerBlock * self.nTxs, self.processingHeaderObj.nHeights / self.nTxs))
310
317
311 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
318 self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock * self.nTxs
312 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights/self.nTxs) *self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
319 self.dataOut.heightList = numpy.arange(self.processingHeaderObj.nHeights / self.nTxs) * \
313 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
320 self.processingHeaderObj.deltaHeight + self.processingHeaderObj.firstHeight
321 self.dataOut.radarControllerHeaderObj.ippSeconds = self.radarControllerHeaderObj.ippSeconds / self.nTxs
314
322
315 return
323 return
316
324
317 def readFirstHeaderFromServer(self):
325 def readFirstHeaderFromServer(self):
318
326
319 self.getFirstHeader()
327 self.getFirstHeader()
320
328
321 self.firstHeaderSize = self.basicHeaderObj.size
329 self.firstHeaderSize = self.basicHeaderObj.size
322
330
323 datatype = int(numpy.log2((self.processingHeaderObj.processFlags & PROCFLAG.DATATYPE_MASK))-numpy.log2(PROCFLAG.DATATYPE_CHAR))
331 datatype = int(numpy.log2((self.processingHeaderObj.processFlags &
332 PROCFLAG.DATATYPE_MASK)) - numpy.log2(PROCFLAG.DATATYPE_CHAR))
324 if datatype == 0:
333 if datatype == 0:
325 datatype_str = numpy.dtype([('real','<i1'),('imag','<i1')])
334 datatype_str = numpy.dtype([('real', '<i1'), ('imag', '<i1')])
326 elif datatype == 1:
335 elif datatype == 1:
327 datatype_str = numpy.dtype([('real','<i2'),('imag','<i2')])
336 datatype_str = numpy.dtype([('real', '<i2'), ('imag', '<i2')])
328 elif datatype == 2:
337 elif datatype == 2:
329 datatype_str = numpy.dtype([('real','<i4'),('imag','<i4')])
338 datatype_str = numpy.dtype([('real', '<i4'), ('imag', '<i4')])
330 elif datatype == 3:
339 elif datatype == 3:
331 datatype_str = numpy.dtype([('real','<i8'),('imag','<i8')])
340 datatype_str = numpy.dtype([('real', '<i8'), ('imag', '<i8')])
332 elif datatype == 4:
341 elif datatype == 4:
333 datatype_str = numpy.dtype([('real','<f4'),('imag','<f4')])
342 datatype_str = numpy.dtype([('real', '<f4'), ('imag', '<f4')])
334 elif datatype == 5:
343 elif datatype == 5:
335 datatype_str = numpy.dtype([('real','<f8'),('imag','<f8')])
344 datatype_str = numpy.dtype([('real', '<f8'), ('imag', '<f8')])
336 else:
345 else:
337 raise ValueError, 'Data type was not defined'
346 raise ValueError, 'Data type was not defined'
338
347
339 self.dtype = datatype_str
348 self.dtype = datatype_str
340 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
349 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
341 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
350 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + \
351 self.firstHeaderSize + self.basicHeaderSize * \
352 (self.processingHeaderObj.dataBlocksPerFile - 1)
342 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
353 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
343 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
354 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
344 self.getBlockDimension()
355 self.getBlockDimension()
345
356
346
357 def getFromServer(self):
347 def getFromServer(self):
348 self.flagDiscontinuousBlock = 0
358 self.flagDiscontinuousBlock = 0
349 self.profileIndex = 0
359 self.profileIndex = 0
350 self.flagIsNewBlock = 1
360 self.flagIsNewBlock = 1
@@ -364,38 +374,46 class VoltageReader(JRODataReader, ProcessingUnit):
364 self.processingHeaderObj.read(block[self.blockPointer:])
374 self.processingHeaderObj.read(block[self.blockPointer:])
365 self.blockPointer += self.processingHeaderObj.length
375 self.blockPointer += self.processingHeaderObj.length
366 self.readFirstHeaderFromServer()
376 self.readFirstHeaderFromServer()
367
377
368 timestamp = self.basicHeaderObj.get_datatime()
378 timestamp = self.basicHeaderObj.get_datatime()
369 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
379 print '[Reading] - Block {} - {}'.format(self.nTotalBlocks, timestamp)
370 current_pointer_location = self.blockPointer
380 current_pointer_location = self.blockPointer
371 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
381 junk = numpy.fromstring(
382 block[self.blockPointer:], self.dtype, self.blocksize)
372
383
373 try:
384 try:
374 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
385 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
386 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
375 except:
387 except:
376 #print "The read block (%3d) has not enough data" %self.nReadBlocks
388 # print "The read block (%3d) has not enough data" %self.nReadBlocks
377 if self.waitDataBlock(pointer_location=current_pointer_location):
389 if self.waitDataBlock(pointer_location=current_pointer_location):
378 junk = numpy.fromstring( block[self.blockPointer:], self.dtype, self.blocksize )
390 junk = numpy.fromstring(
379 junk = junk.reshape( (self.processingHeaderObj.profilesPerBlock, self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels) )
391 block[self.blockPointer:], self.dtype, self.blocksize)
392 junk = junk.reshape((self.processingHeaderObj.profilesPerBlock,
393 self.processingHeaderObj.nHeights, self.systemHeaderObj.nChannels))
380 # return 0
394 # return 0
381
395
382 #Dimensions : nChannels, nProfiles, nSamples
396 # Dimensions : nChannels, nProfiles, nSamples
383
397
384 junk = numpy.transpose(junk, (2,0,1))
398 junk = numpy.transpose(junk, (2, 0, 1))
385 self.datablock = junk['real'] + junk['imag'] * 1j
399 self.datablock = junk['real'] + junk['imag'] * 1j
386 self.profileIndex = 0
400 self.profileIndex = 0
387 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
401 if self.selBlocksize == None:
402 self.selBlocksize = self.dataOut.nProfiles
388 if self.selBlocktime != None:
403 if self.selBlocktime != None:
389 if self.dataOut.nCohInt is not None:
404 if self.dataOut.nCohInt is not None:
390 nCohInt = self.dataOut.nCohInt
405 nCohInt = self.dataOut.nCohInt
391 else:
406 else:
392 nCohInt = 1
407 nCohInt = 1
393 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
408 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
394 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
409 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
410 self.dataOut.data = self.datablock[:,
411 self.profileIndex:self.profileIndex + self.selBlocksize, :]
395 datasize = self.dataOut.data.shape[1]
412 datasize = self.dataOut.data.shape[1]
396 if datasize < self.selBlocksize:
413 if datasize < self.selBlocksize:
397 buffer = numpy.zeros((self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype = 'complex')
414 buffer = numpy.zeros(
398 buffer[:,:datasize,:] = self.dataOut.data
415 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
416 buffer[:, :datasize, :] = self.dataOut.data
399 self.dataOut.data = buffer
417 self.dataOut.data = buffer
400 self.profileIndex = blockIndex
418 self.profileIndex = blockIndex
401
419
@@ -411,30 +429,30 class VoltageReader(JRODataReader, ProcessingUnit):
411 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
429 del tipo "Voltage" con todos los parametros asociados a este (metadata). cuando no hay datos
412 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
430 en el buffer de lectura es necesario hacer una nueva lectura de los bloques de datos usando
413 "readNextBlock"
431 "readNextBlock"
414
432
415 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
433 Ademas incrementa el contador del buffer "self.profileIndex" en 1.
416
434
417 Return:
435 Return:
418
436
419 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
437 Si el flag self.getByBlock ha sido seteado el bloque completo es copiado a self.dataOut y el self.profileIndex
420 es igual al total de perfiles leidos desde el archivo.
438 es igual al total de perfiles leidos desde el archivo.
421
439
422 Si self.getByBlock == False:
440 Si self.getByBlock == False:
423
441
424 self.dataOut.data = buffer[:, thisProfile, :]
442 self.dataOut.data = buffer[:, thisProfile, :]
425
443
426 shape = [nChannels, nHeis]
444 shape = [nChannels, nHeis]
427
445
428 Si self.getByBlock == True:
446 Si self.getByBlock == True:
429
447
430 self.dataOut.data = buffer[:, :, :]
448 self.dataOut.data = buffer[:, :, :]
431
449
432 shape = [nChannels, nProfiles, nHeis]
450 shape = [nChannels, nProfiles, nHeis]
433
451
434 Variables afectadas:
452 Variables afectadas:
435 self.dataOut
453 self.dataOut
436 self.profileIndex
454 self.profileIndex
437
455
438 Affected:
456 Affected:
439 self.dataOut
457 self.dataOut
440 self.profileIndex
458 self.profileIndex
@@ -448,7 +466,7 class VoltageReader(JRODataReader, ProcessingUnit):
448 self.flagDiscontinuousBlock = 0
466 self.flagDiscontinuousBlock = 0
449 self.flagIsNewBlock = 0
467 self.flagIsNewBlock = 0
450 if self.__hasNotDataInBuffer():
468 if self.__hasNotDataInBuffer():
451 if not( self.readNextBlock() ):
469 if not(self.readNextBlock()):
452 return 0
470 return 0
453
471
454 self.getFirstHeader()
472 self.getFirstHeader()
@@ -467,11 +485,11 class VoltageReader(JRODataReader, ProcessingUnit):
467 blocks is increased by nTxs (nProfiles *= nTxs)
485 blocks is increased by nTxs (nProfiles *= nTxs)
468 """
486 """
469 self.dataOut.flagDataAsBlock = False
487 self.dataOut.flagDataAsBlock = False
470 self.dataOut.data = self.datablock[:,self.profileIndex,:]
488 self.dataOut.data = self.datablock[:, self.profileIndex, :]
471 self.dataOut.profileIndex = self.profileIndex
489 self.dataOut.profileIndex = self.profileIndex
472
490
473 self.profileIndex += 1
491 self.profileIndex += 1
474
492
475 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
493 # elif self.selBlocksize==None or self.selBlocksize==self.dataOut.nProfiles:
476 # """
494 # """
477 # Return all block
495 # Return all block
@@ -479,42 +497,47 class VoltageReader(JRODataReader, ProcessingUnit):
479 # self.dataOut.flagDataAsBlock = True
497 # self.dataOut.flagDataAsBlock = True
480 # self.dataOut.data = self.datablock
498 # self.dataOut.data = self.datablock
481 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
499 # self.dataOut.profileIndex = self.dataOut.nProfiles - 1
482 #
500 #
483 # self.profileIndex = self.dataOut.nProfiles
501 # self.profileIndex = self.dataOut.nProfiles
484
502
485 else:
503 else:
486 """
504 """
487 Return a block
505 Return a block
488 """
506 """
489 if self.selBlocksize == None: self.selBlocksize = self.dataOut.nProfiles
507 if self.selBlocksize == None:
508 self.selBlocksize = self.dataOut.nProfiles
490 if self.selBlocktime != None:
509 if self.selBlocktime != None:
491 if self.dataOut.nCohInt is not None:
510 if self.dataOut.nCohInt is not None:
492 nCohInt = self.dataOut.nCohInt
511 nCohInt = self.dataOut.nCohInt
493 else:
512 else:
494 nCohInt = 1
513 nCohInt = 1
495 self.selBlocksize = int(self.dataOut.nProfiles*round(self.selBlocktime/(nCohInt*self.dataOut.ippSeconds*self.dataOut.nProfiles)))
514 self.selBlocksize = int(self.dataOut.nProfiles * round(self.selBlocktime / (
515 nCohInt * self.dataOut.ippSeconds * self.dataOut.nProfiles)))
496
516
497 self.dataOut.data = self.datablock[:,self.profileIndex:self.profileIndex+self.selBlocksize,:]
517 self.dataOut.data = self.datablock[:,
518 self.profileIndex:self.profileIndex + self.selBlocksize, :]
498 self.profileIndex += self.selBlocksize
519 self.profileIndex += self.selBlocksize
499 datasize = self.dataOut.data.shape[1]
520 datasize = self.dataOut.data.shape[1]
500
521
501 if datasize < self.selBlocksize:
522 if datasize < self.selBlocksize:
502 buffer = numpy.zeros((self.dataOut.data.shape[0],self.selBlocksize,self.dataOut.data.shape[2]), dtype = 'complex')
523 buffer = numpy.zeros(
503 buffer[:,:datasize,:] = self.dataOut.data
524 (self.dataOut.data.shape[0], self.selBlocksize, self.dataOut.data.shape[2]), dtype='complex')
525 buffer[:, :datasize, :] = self.dataOut.data
504
526
505 while datasize < self.selBlocksize: #Not enough profiles to fill the block
527 while datasize < self.selBlocksize: # Not enough profiles to fill the block
506 if not( self.readNextBlock() ):
528 if not(self.readNextBlock()):
507 return 0
529 return 0
508 self.getFirstHeader()
530 self.getFirstHeader()
509 self.reshapeData()
531 self.reshapeData()
510 if self.datablock is None:
532 if self.datablock is None:
511 self.dataOut.flagNoData = True
533 self.dataOut.flagNoData = True
512 return 0
534 return 0
513 #stack data
535 # stack data
514 blockIndex = self.selBlocksize - datasize
536 blockIndex = self.selBlocksize - datasize
515 datablock1 = self.datablock[:,:blockIndex,:]
537 datablock1 = self.datablock[:, :blockIndex, :]
516
538
517 buffer[:,datasize:datasize+datablock1.shape[1],:] = datablock1
539 buffer[:, datasize:datasize +
540 datablock1.shape[1], :] = datablock1
518 datasize += datablock1.shape[1]
541 datasize += datablock1.shape[1]
519
542
520 self.dataOut.data = buffer
543 self.dataOut.data = buffer
@@ -531,6 +554,7 class VoltageReader(JRODataReader, ProcessingUnit):
531
554
532 return self.dataOut.data
555 return self.dataOut.data
533
556
557
534 class VoltageWriter(JRODataWriter, Operation):
558 class VoltageWriter(JRODataWriter, Operation):
535 """
559 """
536 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
560 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
@@ -543,7 +567,6 class VoltageWriter(JRODataWriter, Operation):
543
567
544 shapeBuffer = None
568 shapeBuffer = None
545
569
546
547 def __init__(self, **kwargs):
570 def __init__(self, **kwargs):
548 """
571 """
549 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
572 Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
@@ -590,7 +613,6 class VoltageWriter(JRODataWriter, Operation):
590 return 1
613 return 1
591 return 0
614 return 0
592
615
593
594 def setBlockDimension(self):
616 def setBlockDimension(self):
595 """
617 """
596 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
618 Obtiene las formas dimensionales del los subbloques de datos que componen un bloque
@@ -607,8 +629,8 class VoltageWriter(JRODataWriter, Operation):
607 self.systemHeaderObj.nChannels)
629 self.systemHeaderObj.nChannels)
608
630
609 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
631 self.datablock = numpy.zeros((self.systemHeaderObj.nChannels,
610 self.processingHeaderObj.profilesPerBlock,
632 self.processingHeaderObj.profilesPerBlock,
611 self.processingHeaderObj.nHeights),
633 self.processingHeaderObj.nHeights),
612 dtype=numpy.dtype('complex64'))
634 dtype=numpy.dtype('complex64'))
613
635
614 def writeBlock(self):
636 def writeBlock(self):
@@ -624,16 +646,16 class VoltageWriter(JRODataWriter, Operation):
624
646
625 Return: None
647 Return: None
626 """
648 """
627 data = numpy.zeros( self.shapeBuffer, self.dtype )
649 data = numpy.zeros(self.shapeBuffer, self.dtype)
628
650
629 junk = numpy.transpose(self.datablock, (1,2,0))
651 junk = numpy.transpose(self.datablock, (1, 2, 0))
630
652
631 data['real'] = junk.real
653 data['real'] = junk.real
632 data['imag'] = junk.imag
654 data['imag'] = junk.imag
633
655
634 data = data.reshape( (-1) )
656 data = data.reshape((-1))
635
657
636 data.tofile( self.fp )
658 data.tofile(self.fp)
637
659
638 self.datablock.fill(0)
660 self.datablock.fill(0)
639
661
@@ -671,12 +693,12 class VoltageWriter(JRODataWriter, Operation):
671 if self.profileIndex == 0:
693 if self.profileIndex == 0:
672 self.setBasicHeader()
694 self.setBasicHeader()
673
695
674 self.datablock[:,self.profileIndex,:] = self.dataOut.data
696 self.datablock[:, self.profileIndex, :] = self.dataOut.data
675
697
676 self.profileIndex += 1
698 self.profileIndex += 1
677
699
678 if self.hasAllDataInBuffer():
700 if self.hasAllDataInBuffer():
679 #if self.flagIsNewFile:
701 # if self.flagIsNewFile:
680 self.writeNextBlock()
702 self.writeNextBlock()
681 # self.setFirstHeader()
703 # self.setFirstHeader()
682
704
@@ -689,12 +711,12 class VoltageWriter(JRODataWriter, Operation):
689
711
690 dtype_width = self.getDtypeWidth()
712 dtype_width = self.getDtypeWidth()
691
713
692 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels * self.profilesPerBlock * dtype_width * 2)
714 blocksize = int(self.dataOut.nHeights * self.dataOut.nChannels *
715 self.profilesPerBlock * dtype_width * 2)
693
716
694 return blocksize
717 return blocksize
695
718
696 def setFirstHeader(self):
719 def setFirstHeader(self):
697
698 """
720 """
699 Obtiene una copia del First Header
721 Obtiene una copia del First Header
700
722
@@ -711,14 +733,17 class VoltageWriter(JRODataWriter, Operation):
711 self.systemHeaderObj.nChannels = self.dataOut.nChannels
733 self.systemHeaderObj.nChannels = self.dataOut.nChannels
712 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
734 self.radarControllerHeaderObj = self.dataOut.radarControllerHeaderObj.copy()
713
735
714 self.processingHeaderObj.dtype = 0 # Voltage
736 self.processingHeaderObj.dtype = 0 # Voltage
715 self.processingHeaderObj.blockSize = self.__getBlockSize()
737 self.processingHeaderObj.blockSize = self.__getBlockSize()
716 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
738 self.processingHeaderObj.profilesPerBlock = self.profilesPerBlock
717 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
739 self.processingHeaderObj.dataBlocksPerFile = self.blocksPerFile
718 self.processingHeaderObj.nWindows = 1 #podria ser 1 o self.dataOut.processingHeaderObj.nWindows
740 # podria ser 1 o self.dataOut.processingHeaderObj.nWindows
741 self.processingHeaderObj.nWindows = 1
719 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
742 self.processingHeaderObj.nCohInt = self.dataOut.nCohInt
720 self.processingHeaderObj.nIncohInt = 1 # Cuando la data de origen es de tipo Voltage
743 # Cuando la data de origen es de tipo Voltage
721 self.processingHeaderObj.totalSpectra = 0 # Cuando la data de origen es de tipo Voltage
744 self.processingHeaderObj.nIncohInt = 1
745 # Cuando la data de origen es de tipo Voltage
746 self.processingHeaderObj.totalSpectra = 0
722
747
723 if self.dataOut.code is not None:
748 if self.dataOut.code is not None:
724 self.processingHeaderObj.code = self.dataOut.code
749 self.processingHeaderObj.code = self.dataOut.code
@@ -727,7 +752,8 class VoltageWriter(JRODataWriter, Operation):
727
752
728 if self.processingHeaderObj.nWindows != 0:
753 if self.processingHeaderObj.nWindows != 0:
729 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
754 self.processingHeaderObj.firstHeight = self.dataOut.heightList[0]
730 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
755 self.processingHeaderObj.deltaHeight = self.dataOut.heightList[1] - \
756 self.dataOut.heightList[0]
731 self.processingHeaderObj.nHeights = self.dataOut.nHeights
757 self.processingHeaderObj.nHeights = self.dataOut.nHeights
732 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
758 self.processingHeaderObj.samplesWin = self.dataOut.nHeights
733
759
General Comments 0
You need to be logged in to leave comments. Login now