##// END OF EJS Templates
schain-jc
RSS
Change multiSchain by MPProject
Juan C. Espinoza -
r1052:3fb7c359028f
parent child
Show More
Side by Side Unified
Context file:
r1052:3fb7c359028f Loading diff...:
Show file before | Show file after | Hide comments
@@ -9,56 +9,53 import datetime
9 import traceback
9 import traceback
10 import math
10 import math
11 import time
11 import time
12 from multiprocessing import Process, Queue, cpu_count
12 from multiprocessing import Process, cpu_count
13
14 import schainpy
15 import schainpy.admin
16 from schainpy.utils.log import logToFile
17
13
18 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 from xml.dom import minidom
15 from xml.dom import minidom
20
16
17 import schainpy
18 import schainpy.admin
21 from schainpy.model import *
19 from schainpy.model import *
22 from time import sleep
20 from schainpy.utils import log
23
21
24
22 DTYPES = {
25
23 'Voltage': '.r',
26 def prettify(elem):
24 'Spectra': '.pdata'
27 """Return a pretty-printed XML string for the Element.
25 }
28 """
26
29 rough_string = tostring(elem, 'utf-8')
27 def MPProject(project, n=cpu_count()):
30 reparsed = minidom.parseString(rough_string)
28 '''
31 return reparsed.toprettyxml(indent=" ")
29 Project wrapper to run schain in n processes
32
30 '''
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
31
34 skip = 0
32 rconf = project.getReadUnitObj()
35 cursor = 0
33 op = rconf.getOperationObj('run')
36 nFiles = None
34 dt1 = op.getParameterValue('startDate')
37 processes = []
35 dt2 = op.getParameterValue('endDate')
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
40 days = (dt2 - dt1).days
36 days = (dt2 - dt1).days
41
37
42 for day in range(days+1):
38 for day in range(days+1):
43 skip = 0
39 skip = 0
44 cursor = 0
40 cursor = 0
45 q = Queue()
46 processes = []
41 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
42 dt = dt1 + datetime.timedelta(day)
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
43 dt_str = dt.strftime('%Y/%m/%d')
49 firstProcess.start()
44 reader = JRODataReader()
50 if by_day:
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
51 continue
46 startDate=dt,
52 nFiles = q.get()
47 endDate=dt,
53 if nFiles==0:
48 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
50 if nFiles == 0:
54 continue
51 continue
55 firstProcess.terminate()
52 skip = int(math.ceil(nFiles/n))
56 skip = int(math.ceil(nFiles/nProcess))
53 while nFiles > cursor*skip:
57 while True:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
55 skip=skip)
59 processes[cursor].start()
56 p = project.clone()
60 if nFiles < cursor*skip:
57 p.start()
61 break
58 processes.append(p)
62 cursor += 1
59 cursor += 1
63
60
64 def beforeExit(exctype, value, trace):
61 def beforeExit(exctype, value, trace):
@@ -75,7 +72,6 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_da
75
72
76 time.sleep(3)
73 time.sleep(3)
77
74
78
79 class ParameterConf():
75 class ParameterConf():
80
76
81 id = None
77 id = None
@@ -112,7 +108,7 class ParameterConf():
112 return self.__formated_value
108 return self.__formated_value
113
109
114 if value == '':
110 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
111 raise ValueError, '%s: This parameter value is empty' %self.name
116
112
117 if format == 'list':
113 if format == 'list':
118 strList = value.split(',')
114 strList = value.split(',')
@@ -122,10 +118,10 class ParameterConf():
122 return self.__formated_value
118 return self.__formated_value
123
119
124 if format == 'intlist':
120 if format == 'intlist':
125 """
121 '''
126 Example:
122 Example:
127 value = (0,1,2)
123 value = (0,1,2)
128 """
124 '''
129
125
130 new_value = ast.literal_eval(value)
126 new_value = ast.literal_eval(value)
131
127
@@ -137,10 +133,10 class ParameterConf():
137 return self.__formated_value
133 return self.__formated_value
138
134
139 if format == 'floatlist':
135 if format == 'floatlist':
140 """
136 '''
141 Example:
137 Example:
142 value = (0.5, 1.4, 2.7)
138 value = (0.5, 1.4, 2.7)
143 """
139 '''
144
140
145 new_value = ast.literal_eval(value)
141 new_value = ast.literal_eval(value)
146
142
@@ -170,38 +166,38 class ParameterConf():
170 return self.__formated_value
166 return self.__formated_value
171
167
172 if format == 'pairslist':
168 if format == 'pairslist':
173 """
169 '''
174 Example:
170 Example:
175 value = (0,1),(1,2)
171 value = (0,1),(1,2)
176 """
172 '''
177
173
178 new_value = ast.literal_eval(value)
174 new_value = ast.literal_eval(value)
179
175
180 if type(new_value) not in (tuple, list):
176 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
182
178
183 if type(new_value[0]) not in (tuple, list):
179 if type(new_value[0]) not in (tuple, list):
184 if len(new_value) != 2:
180 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
186 new_value = [new_value]
182 new_value = [new_value]
187
183
188 for thisPair in new_value:
184 for thisPair in new_value:
189 if len(thisPair) != 2:
185 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
191
187
192 self.__formated_value = new_value
188 self.__formated_value = new_value
193
189
194 return self.__formated_value
190 return self.__formated_value
195
191
196 if format == 'multilist':
192 if format == 'multilist':
197 """
193 '''
198 Example:
194 Example:
199 value = (0,1,2),(3,4,5)
195 value = (0,1,2),(3,4,5)
200 """
196 '''
201 multiList = ast.literal_eval(value)
197 multiList = ast.literal_eval(value)
202
198
203 if type(multiList[0]) == int:
199 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
200 multiList = ast.literal_eval('(' + value + ')')
205
201
206 self.__formated_value = multiList
202 self.__formated_value = multiList
207
203
@@ -263,9 +259,9 class ParameterConf():
263
259
264 def printattr(self):
260 def printattr(self):
265
261
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
267
263
268 class OperationConf():
264 class OperationConf():
269
265
270 id = None
266 id = None
271 name = None
267 name = None
@@ -371,7 +367,9 class OperationConf():
371 self.parmConfObjList = []
367 self.parmConfObjList = []
372
368
373 def addParameter(self, name, value, format='str'):
369 def addParameter(self, name, value, format='str'):
374
370
371 if value is None:
372 return None
375 id = self.__getNewId()
373 id = self.__getNewId()
376
374
377 parmConfObj = ParameterConf()
375 parmConfObj = ParameterConf()
@@ -431,7 +429,7 class OperationConf():
431
429
432 def printattr(self):
430 def printattr(self):
433
431
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
435 self.id,
433 self.id,
436 self.name,
434 self.name,
437 self.type,
435 self.type,
@@ -444,12 +442,11 class OperationConf():
444
442
445
443
446 if self.type == 'self':
444 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
445 raise ValueError, 'This operation type cannot be created'
448
446
449 if self.type == 'plotter':
447 if self.type == 'plotter':
450 #Plotter(plotter_name)
451 if not plotter_queue:
448 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453
450
454 opObj = Plotter(self.name, plotter_queue)
451 opObj = Plotter(self.name, plotter_queue)
455
452
@@ -564,7 +561,7 class ProcUnitConf():
564
561
565 #Compatible with old signal chain version
562 #Compatible with old signal chain version
566 if datatype==None and name==None:
563 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
564 raise ValueError, 'datatype or name should be defined'
568
565
569 if name==None:
566 if name==None:
570 if 'Proc' in datatype:
567 if 'Proc' in datatype:
@@ -595,7 +592,7 class ProcUnitConf():
595
592
596 def addParameter(self, **kwargs):
593 def addParameter(self, **kwargs):
597 '''
594 '''
598 Add parameters to "run" operation
595 Add parameters to 'run' operation
599 '''
596 '''
600 opObj = self.opConfObjList[0]
597 opObj = self.opConfObjList[0]
601
598
@@ -633,11 +630,11 class ProcUnitConf():
633 self.datatype = upElement.get('datatype')
630 self.datatype = upElement.get('datatype')
634 self.inputId = upElement.get('inputId')
631 self.inputId = upElement.get('inputId')
635
632
636 if self.ELEMENTNAME == "ReadUnit":
633 if self.ELEMENTNAME == 'ReadUnit':
637 self.datatype = self.datatype.replace("Reader", "")
634 self.datatype = self.datatype.replace('Reader', '')
638
635
639 if self.ELEMENTNAME == "ProcUnit":
636 if self.ELEMENTNAME == 'ProcUnit':
640 self.datatype = self.datatype.replace("Proc", "")
637 self.datatype = self.datatype.replace('Proc', '')
641
638
642 if self.inputId == 'None':
639 if self.inputId == 'None':
643 self.inputId = '0'
640 self.inputId = '0'
@@ -653,7 +650,7 class ProcUnitConf():
653
650
654 def printattr(self):
651 def printattr(self):
655
652
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
657 self.id,
654 self.id,
658 self.name,
655 self.name,
659 self.datatype,
656 self.datatype,
@@ -707,17 +704,9 class ProcUnitConf():
707
704
708 kwargs[parmConfObj.name] = parmConfObj.getValue()
705 kwargs[parmConfObj.name] = parmConfObj.getValue()
709
706
710 #ini = time.time()
711
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
713 sts = self.procUnitObj.call(opType = opConfObj.type,
707 sts = self.procUnitObj.call(opType = opConfObj.type,
714 opName = opConfObj.name,
708 opName = opConfObj.name,
715 opId = opConfObj.id)
709 opId = opConfObj.id)
716
717 # total_time = time.time() - ini
718 #
719 # if total_time > 0.002:
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
721
710
722 is_ok = is_ok or sts
711 is_ok = is_ok or sts
723
712
@@ -762,11 +751,12 class ReadUnitConf(ProcUnitConf):
762
751
763 return self.ELEMENTNAME
752 return self.ELEMENTNAME
764
753
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
767 #Compatible with old signal chain version
757 #Compatible with old signal chain version
768 if datatype==None and name==None:
758 if datatype==None and name==None:
769 raise ValueError, "datatype or name should be defined"
759 raise ValueError, 'datatype or name should be defined'
770
760
771 if name==None:
761 if name==None:
772 if 'Reader' in datatype:
762 if 'Reader' in datatype:
@@ -785,39 +775,28 class ReadUnitConf(ProcUnitConf):
785 self.endDate = endDate
775 self.endDate = endDate
786 self.startTime = startTime
776 self.startTime = startTime
787 self.endTime = endTime
777 self.endTime = endTime
788
789 self.inputId = '0'
778 self.inputId = '0'
790 self.parentId = parentId
779 self.parentId = parentId
791 self.queue = queue
792 self.server = server
780 self.server = server
793 self.addRunOperation(**kwargs)
781 self.addRunOperation(**kwargs)
794
782
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
783 def update(self, **kwargs):
796
784
797 #Compatible with old signal chain version
785 if 'datatype' in kwargs:
798 if datatype==None and name==None:
786 datatype = kwargs.pop('datatype')
799 raise ValueError, "datatype or name should be defined"
800
801 if name==None:
802 if 'Reader' in datatype:
787 if 'Reader' in datatype:
803 name = datatype
788 self.name = datatype
804 else:
789 else:
805 name = '%sReader' %(datatype)
790 self.name = '%sReader' %(datatype)
806
791 self.datatype = self.name.replace('Reader', '')
807 if datatype==None:
808 datatype = name.replace('Reader','')
809
810 self.datatype = datatype
811 self.name = name
812 self.path = path
813 self.startDate = startDate
814 self.endDate = endDate
815 self.startTime = startTime
816 self.endTime = endTime
817
792
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
794
795 for attr in attrs:
796 if attr in kwargs:
797 setattr(self, attr, kwargs.pop(attr))
798
818 self.inputId = '0'
799 self.inputId = '0'
819 self.parentId = parentId
820
821 self.updateRunOperation(**kwargs)
800 self.updateRunOperation(**kwargs)
822
801
823 def removeOperations(self):
802 def removeOperations(self):
@@ -832,13 +811,13 class ReadUnitConf(ProcUnitConf):
832 opObj = self.addOperation(name = 'run', optype = 'self')
811 opObj = self.addOperation(name = 'run', optype = 'self')
833
812
834 if self.server is None:
813 if self.server is None:
835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
836 opObj.addParameter(name='path' , value=self.path, format='str')
815 opObj.addParameter(name='path', value=self.path, format='str')
837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
820
842 for key, value in kwargs.items():
821 for key, value in kwargs.items():
843 opObj.addParameter(name=key, value=value, format=type(value).__name__)
822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
844 else:
823 else:
@@ -849,32 +828,21 class ReadUnitConf(ProcUnitConf):
849
828
850 def updateRunOperation(self, **kwargs):
829 def updateRunOperation(self, **kwargs):
851
830
852 opObj = self.getOperationObj(name = 'run')
831 opObj = self.getOperationObj(name='run')
853 opObj.removeParameters()
832 opObj.removeParameters()
854
833
855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
834 opObj.addParameter(name='datatype', value=self.datatype, format='str')
856 opObj.addParameter(name='path' , value=self.path, format='str')
835 opObj.addParameter(name='path', value=self.path, format='str')
857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
837 opObj.addParameter(name='endDate', value=self.endDate, format='date')
859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
839 opObj.addParameter(name='endTime', value=self.endTime, format='time')
861
840
862 for key, value in kwargs.items():
841 for key, value in kwargs.items():
863 opObj.addParameter(name=key, value=value, format=type(value).__name__)
842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
864
843
865 return opObj
844 return opObj
866
845
867 # def makeXml(self, projectElement):
868 #
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
870 # procUnitElement.set('id', str(self.id))
871 # procUnitElement.set('name', self.name)
872 # procUnitElement.set('datatype', self.datatype)
873 # procUnitElement.set('inputId', str(self.inputId))
874 #
875 # for opConfObj in self.opConfObjList:
876 # opConfObj.makeXml(procUnitElement)
877
878 def readXml(self, upElement):
846 def readXml(self, upElement):
879
847
880 self.id = upElement.get('id')
848 self.id = upElement.get('id')
@@ -882,8 +850,8 class ReadUnitConf(ProcUnitConf):
882 self.datatype = upElement.get('datatype')
850 self.datatype = upElement.get('datatype')
883 self.inputId = upElement.get('inputId')
851 self.inputId = upElement.get('inputId')
884
852
885 if self.ELEMENTNAME == "ReadUnit":
853 if self.ELEMENTNAME == 'ReadUnit':
886 self.datatype = self.datatype.replace("Reader", "")
854 self.datatype = self.datatype.replace('Reader', '')
887
855
888 if self.inputId == 'None':
856 if self.inputId == 'None':
889 self.inputId = '0'
857 self.inputId = '0'
@@ -905,8 +873,9 class ReadUnitConf(ProcUnitConf):
905 self.endTime = opConfObj.getParameterValue('endTime')
873 self.endTime = opConfObj.getParameterValue('endTime')
906
874
907 class Project(Process):
875 class Project(Process):
876
908 id = None
877 id = None
909 name = None
878 # name = None
910 description = None
879 description = None
911 filename = None
880 filename = None
912
881
@@ -916,17 +885,17 class Project(Process):
916
885
917 plotterQueue = None
886 plotterQueue = None
918
887
919 def __init__(self, plotter_queue=None, logfile=None):
888 def __init__(self, plotter_queue=None):
889
920 Process.__init__(self)
890 Process.__init__(self)
921 self.id = None
891 self.id = None
922 self.name = None
892 # self.name = None
923 self.description = None
893 self.description = None
924 if logfile is not None:
894
925 logToFile(logfile)
926 self.plotterQueue = plotter_queue
895 self.plotterQueue = plotter_queue
927
896
928 self.procUnitConfObjDict = {}
897 self.procUnitConfObjDict = {}
929
898
930 def __getNewId(self):
899 def __getNewId(self):
931
900
932 idList = self.procUnitConfObjDict.keys()
901 idList = self.procUnitConfObjDict.keys()
@@ -972,18 +941,28 class Project(Process):
972
941
973 self.procUnitConfObjDict = newProcUnitConfObjDict
942 self.procUnitConfObjDict = newProcUnitConfObjDict
974
943
975 def setup(self, id, name, description):
944 def setup(self, id, name='', description=''):
976
945
946 print
947 print '*'*60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
950 print
977 self.id = str(id)
951 self.id = str(id)
978 self.name = name
979 self.description = description
952 self.description = description
980
953
981 def update(self, name, description):
954 def update(self, name, description):
982
955
983 self.name = name
984 self.description = description
956 self.description = description
985
957
958 def clone(self):
959
960 p = Project()
961 p.procUnitConfObjDict = self.procUnitConfObjDict
962 return p
963
986 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
964 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965
987 if id is None:
966 if id is None:
988 idReadUnit = self.__getNewId()
967 idReadUnit = self.__getNewId()
989 else:
968 else:
@@ -1021,7 +1000,7 class Project(Process):
1021 def getReadUnitObj(self):
1000 def getReadUnitObj(self):
1022
1001
1023 for obj in self.procUnitConfObjDict.values():
1002 for obj in self.procUnitConfObjDict.values():
1024 if obj.getElementName() == "ReadUnit":
1003 if obj.getElementName() == 'ReadUnit':
1025 return obj
1004 return obj
1026
1005
1027 return None
1006 return None
@@ -1066,20 +1045,20 class Project(Process):
1066 if self.filename:
1045 if self.filename:
1067 filename = self.filename
1046 filename = self.filename
1068 else:
1047 else:
1069 filename = "schain.xml"
1048 filename = 'schain.xml'
1070
1049
1071 if not filename:
1050 if not filename:
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1051 print 'filename has not been defined. Use setFilename(filename) for do it.'
1073 return 0
1052 return 0
1074
1053
1075 abs_file = os.path.abspath(filename)
1054 abs_file = os.path.abspath(filename)
1076
1055
1077 if not os.access(os.path.dirname(abs_file), os.W_OK):
1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1079 return 0
1058 return 0
1080
1059
1081 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1082 print "File %s already exists and it could not be overwriten" %abs_file
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1083 return 0
1062 return 0
1084
1063
1085 self.makeXml()
1064 self.makeXml()
@@ -1093,13 +1072,13 class Project(Process):
1093 def readXml(self, filename = None):
1072 def readXml(self, filename = None):
1094
1073
1095 if not filename:
1074 if not filename:
1096 print "filename is not defined"
1075 print 'filename is not defined'
1097 return 0
1076 return 0
1098
1077
1099 abs_file = os.path.abspath(filename)
1078 abs_file = os.path.abspath(filename)
1100
1079
1101 if not os.path.isfile(abs_file):
1080 if not os.path.isfile(abs_file):
1102 print "%s file does not exist" %abs_file
1081 print '%s file does not exist' %abs_file
1103 return 0
1082 return 0
1104
1083
1105 self.projectElement = None
1084 self.projectElement = None
@@ -1108,7 +1087,7 class Project(Process):
1108 try:
1087 try:
1109 self.projectElement = ElementTree().parse(abs_file)
1088 self.projectElement = ElementTree().parse(abs_file)
1110 except:
1089 except:
1111 print "Error reading %s, verify file format" %filename
1090 print 'Error reading %s, verify file format' %filename
1112 return 0
1091 return 0
1113
1092
1114 self.project = self.projectElement.tag
1093 self.project = self.projectElement.tag
@@ -1145,10 +1124,10 class Project(Process):
1145
1124
1146 def printattr(self):
1125 def printattr(self):
1147
1126
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1149 self.name,
1128 self.name,
1150 self.description)
1129 self.description)
1151
1130
1152 for procUnitConfObj in self.procUnitConfObjDict.values():
1131 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 procUnitConfObj.printattr()
1132 procUnitConfObj.printattr()
1154
1133
@@ -1179,7 +1158,7 class Project(Process):
1179
1158
1180 self.__connect(puObjIN, thisPUObj)
1159 self.__connect(puObjIN, thisPUObj)
1181
1160
1182 def __handleError(self, procUnitConfObj, send_email=True):
1161 def __handleError(self, procUnitConfObj, send_email=False):
1183
1162
1184 import socket
1163 import socket
1185
1164
@@ -1187,33 +1166,33 class Project(Process):
1187 sys.exc_info()[1],
1166 sys.exc_info()[1],
1188 sys.exc_info()[2])
1167 sys.exc_info()[2])
1189
1168
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1191 print "***** %s" %err[-1]
1170 print '***** %s' %err[-1]
1192
1171
1193 message = "".join(err)
1172 message = ''.join(err)
1194
1173
1195 sys.stderr.write(message)
1174 sys.stderr.write(message)
1196
1175
1197 if not send_email:
1176 if not send_email:
1198 return
1177 return
1199
1178
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1201
1180
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1205 subtitle += "Configuration file: %s\n" %self.filename
1184 subtitle += 'Configuration file: %s\n' %self.filename
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1207
1186
1208 readUnitConfObj = self.getReadUnitObj()
1187 readUnitConfObj = self.getReadUnitObj()
1209 if readUnitConfObj:
1188 if readUnitConfObj:
1210 subtitle += "\nInput parameters:\n"
1189 subtitle += '\nInput parameters:\n'
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1217
1196
1218 adminObj = schainpy.admin.SchainNotify()
1197 adminObj = schainpy.admin.SchainNotify()
1219 adminObj.sendAlert(message=message,
1198 adminObj.sendAlert(message=message,
@@ -1228,15 +1207,15 class Project(Process):
1228 return 0
1207 return 0
1229
1208
1230 def runController(self):
1209 def runController(self):
1231 """
1210 '''
1232 returns 0 when this process has been stopped, 1 otherwise
1211 returns 0 when this process has been stopped, 1 otherwise
1233 """
1212 '''
1234
1213
1235 if self.isPaused():
1214 if self.isPaused():
1236 print "Process suspended"
1215 print 'Process suspended'
1237
1216
1238 while True:
1217 while True:
1239 sleep(0.1)
1218 time.sleep(0.1)
1240
1219
1241 if not self.isPaused():
1220 if not self.isPaused():
1242 break
1221 break
@@ -1244,10 +1223,10 class Project(Process):
1244 if self.isStopped():
1223 if self.isStopped():
1245 break
1224 break
1246
1225
1247 print "Process reinitialized"
1226 print 'Process reinitialized'
1248
1227
1249 if self.isStopped():
1228 if self.isStopped():
1250 print "Process stopped"
1229 print 'Process stopped'
1251 return 0
1230 return 0
1252
1231
1253 return 1
1232 return 1
@@ -1258,29 +1237,23 class Project(Process):
1258
1237
1259 def setPlotterQueue(self, plotter_queue):
1238 def setPlotterQueue(self, plotter_queue):
1260
1239
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1240 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1262
1241
1263 def getPlotterQueue(self):
1242 def getPlotterQueue(self):
1264
1243
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1244 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1266
1245
1267 def useExternalPlotter(self):
1246 def useExternalPlotter(self):
1268
1247
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1248 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270
1249
1250 def run(self):
1271
1251
1272 def run(self, filename=None):
1252 log.success('Starting {}'.format(self.name))
1273
1253
1274 # self.writeXml(filename)
1275 self.createObjects()
1254 self.createObjects()
1276 self.connectObjects()
1255 self.connectObjects()
1277
1256
1278 print
1279 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1282 print
1283
1284 keyList = self.procUnitConfObjDict.keys()
1257 keyList = self.procUnitConfObjDict.keys()
1285 keyList.sort()
1258 keyList.sort()
1286
1259
@@ -1289,7 +1262,6 class Project(Process):
1289 is_ok = False
1262 is_ok = False
1290
1263
1291 for procKey in keyList:
1264 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293
1265
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1266 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295
1267
@@ -1300,19 +1272,18 class Project(Process):
1300 is_ok = False
1272 is_ok = False
1301 break
1273 break
1302 except ValueError, e:
1274 except ValueError, e:
1303 sleep(0.5)
1275 time.sleep(0.5)
1304 self.__handleError(procUnitConfObj, send_email=True)
1276 self.__handleError(procUnitConfObj, send_email=True)
1305 is_ok = False
1277 is_ok = False
1306 break
1278 break
1307 except:
1279 except:
1308 sleep(0.5)
1280 time.sleep(0.5)
1309 self.__handleError(procUnitConfObj)
1281 self.__handleError(procUnitConfObj)
1310 is_ok = False
1282 is_ok = False
1311 break
1283 break
1312
1284
1313 #If every process unit finished so end process
1285 #If every process unit finished so end process
1314 if not(is_ok):
1286 if not(is_ok):
1315 # print "Every process unit have finished"
1316 break
1287 break
1317
1288
1318 if not self.runController():
1289 if not self.runController():
@@ -1322,3 +1293,5 class Project(Process):
1322 for procKey in keyList:
1293 for procKey in keyList:
1323 procUnitConfObj = self.procUnitConfObjDict[procKey]
1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 procUnitConfObj.close()
1295 procUnitConfObj.close()
1296
1297 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
@@ -6218,3 +6218,6 class ShowMeConsole(QtCore.QObject):
6218 text = text[:-1]
6218 text = text[:-1]
6219
6219
6220 self.textWritten.emit(str(text))
6220 self.textWritten.emit(str(text))
6221
6222 def flush(self):
6223 pass
Show file before | Show file after | Hide comments
@@ -267,7 +267,7 class AMISRReader(ProcessingUnit):
267 self.dirnameList = new_dirnameList
267 self.dirnameList = new_dirnameList
268 return 1
268 return 1
269
269
270 def __searchFilesOnline(self,
270 def searchFilesOnLine(self,
271 path,
271 path,
272 walk=True):
272 walk=True):
273
273
@@ -287,7 +287,7 class AMISRReader(ProcessingUnit):
287 return
287 return
288
288
289
289
290 def __searchFilesOffline(self,
290 def searchFilesOffLine(self,
291 path,
291 path,
292 startDate,
292 startDate,
293 endDate,
293 endDate,
@@ -494,9 +494,9 class AMISRReader(ProcessingUnit):
494 self.online = online
494 self.online = online
495 if not(online):
495 if not(online):
496 #Busqueda de archivos offline
496 #Busqueda de archivos offline
497 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 else:
498 else:
499 self.__searchFilesOnline(path, walk)
499 self.searchFilesOnLine(path, walk)
500
500
501 if not(self.filenameList):
501 if not(self.filenameList):
502 print "There is no files into the folder: %s"%(path)
502 print "There is no files into the folder: %s"%(path)
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (543 lines changed) Show them Hide them
@@ -541,8 +541,7 class JRODataIO:
541 return inspect.getargspec(self.run).args
541 return inspect.getargspec(self.run).args
542
542
543 class JRODataReader(JRODataIO):
543 class JRODataReader(JRODataIO):
544
544
545 firstTime = True
546 online = 0
545 online = 0
547
546
548 realtime = 0
547 realtime = 0
@@ -578,8 +577,7 class JRODataReader(JRODataIO):
578 selBlocksize = None
577 selBlocksize = None
579
578
580 selBlocktime = None
579 selBlocktime = None
581
580
582 onlineWithDate = False
583 def __init__(self):
581 def __init__(self):
584
582
585 """
583 """
@@ -603,19 +601,19 class JRODataReader(JRODataIO):
603
601
604 raise NotImplementedError
602 raise NotImplementedError
605
603
606 def __searchFilesOffLine(self,
604 def searchFilesOffLine(self,
607 path,
605 path,
608 startDate=None,
606 startDate=None,
609 endDate=None,
607 endDate=None,
610 startTime=datetime.time(0,0,0),
608 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
609 endTime=datetime.time(23,59,59),
612 set=None,
610 set=None,
613 expLabel='',
611 expLabel='',
614 ext='.r',
612 ext='.r',
615 queue=None,
613 cursor=None,
616 cursor=None,
614 skip=None,
617 skip=None,
615 walk=True):
618 walk=True):
616
619 self.filenameList = []
617 self.filenameList = []
620 self.datetimeList = []
618 self.datetimeList = []
621
619
@@ -624,8 +622,7 class JRODataReader(JRODataIO):
624 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
622 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625
623
626 if dateList == []:
624 if dateList == []:
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
625 return [], []
628 return None, None
629
626
630 if len(dateList) > 1:
627 if len(dateList) > 1:
631 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
@@ -636,18 +633,15 class JRODataReader(JRODataIO):
636 datetimeList = []
633 datetimeList = []
637
634
638 for thisPath in pathList:
635 for thisPath in pathList:
639 # thisPath = pathList[pathDict[file]]
636
640
641 fileList = glob.glob1(thisPath, "*%s" %ext)
637 fileList = glob.glob1(thisPath, "*%s" %ext)
642 fileList.sort()
638 fileList.sort()
643
639
644 skippedFileList = []
640 skippedFileList = []
645
641
646 if cursor is not None and skip is not None:
642 if cursor is not None and skip is not None:
647 # if cursor*skip > len(fileList):
643
648 if skip == 0:
644 if skip == 0:
649 if queue is not None:
650 queue.put(len(fileList))
651 skippedFileList = []
645 skippedFileList = []
652 else:
646 else:
653 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
647 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
@@ -672,44 +666,43 class JRODataReader(JRODataIO):
672
666
673 if not(filenameList):
667 if not(filenameList):
674 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
668 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 return None, None
669 return [], []
676
670
677 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
671 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 print
672 print
679
673
680 for i in range(len(filenameList)):
674 # for i in range(len(filenameList)):
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
675 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682
676
683 self.filenameList = filenameList
677 self.filenameList = filenameList
684 self.datetimeList = datetimeList
678 self.datetimeList = datetimeList
679
685 return pathList, filenameList
680 return pathList, filenameList
686
681
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
682 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
688
683
689 """
684 """
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
685 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 devuelve el archivo encontrado ademas de otros datos.
686 devuelve el archivo encontrado ademas de otros datos.
692
687
693 Input:
688 Input:
694 path : carpeta donde estan contenidos los files que contiene data
689 path : carpeta donde estan contenidos los files que contiene data
695
690
696 expLabel : Nombre del subexperimento (subfolder)
691 expLabel : Nombre del subexperimento (subfolder)
697
692
698 ext : extension de los files
693 ext : extension de los files
699
694
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
695 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
701
696
702 Return:
697 Return:
703 directory : eL directorio donde esta el file encontrado
698 directory : eL directorio donde esta el file encontrado
704 filename : el ultimo file de una determinada carpeta
699 filename : el ultimo file de una determinada carpeta
705 year : el anho
700 year : el anho
706 doy : el numero de dia del anho
701 doy : el numero de dia del anho
707 set : el set del archivo
702 set : el set del archivo
708
703
709
704
710 """
705 """
711 pathList = None
712 filenameList = None
713 if not os.path.isdir(path):
706 if not os.path.isdir(path):
714 return None, None, None, None, None, None
707 return None, None, None, None, None, None
715
708
@@ -719,7 +712,7 class JRODataReader(JRODataIO):
719 fullpath = path
712 fullpath = path
720 foldercounter = 0
713 foldercounter = 0
721 else:
714 else:
722 # Filtra solo los directorios
715 #Filtra solo los directorios
723 for thisPath in os.listdir(path):
716 for thisPath in os.listdir(path):
724 if not os.path.isdir(os.path.join(path,thisPath)):
717 if not os.path.isdir(os.path.join(path,thisPath)):
725 continue
718 continue
@@ -755,7 +748,7 class JRODataReader(JRODataIO):
755
748
756 year = int( filename[1:5] )
749 year = int( filename[1:5] )
757 doy = int( filename[5:8] )
750 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
751 set = int( filename[8:11] )
759
752
760 return fullpath, foldercounter, filename, year, doy, set
753 return fullpath, foldercounter, filename, year, doy, set
761
754
@@ -767,7 +760,7 class JRODataReader(JRODataIO):
767 idFile += 1
760 idFile += 1
768 if not(idFile < len(self.filenameList)):
761 if not(idFile < len(self.filenameList)):
769 self.flagNoMoreFiles = 1
762 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
763 # print "[Reading] No more Files"
771 return 0
764 return 0
772
765
773 filename = self.filenameList[idFile]
766 filename = self.filenameList[idFile]
@@ -785,32 +778,31 class JRODataReader(JRODataIO):
785 self.fileSize = fileSize
778 self.fileSize = fileSize
786 self.fp = fp
779 self.fp = fp
787
780
788 #print "[Reading] Setting the file: %s"%self.filename
781 # print "[Reading] Setting the file: %s"%self.filename
789
782
790 return 1
783 return 1
791
784
792 def __setNextFileOnline(self):
785 def __setNextFileOnline(self):
793 """
786 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
787 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
788 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
789 siguientes.
797
790
798 Affected:
791 Affected:
799 self.flagIsNewFile
792 self.flagIsNewFile
800 self.filename
793 self.filename
801 self.fileSize
794 self.fileSize
802 self.fp
795 self.fp
803 self.set
796 self.set
804 self.flagNoMoreFiles
797 self.flagNoMoreFiles
805
798
806 Return:
799 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
800 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
801 1 : si el file fue abierto con exito y esta listo a ser leido
809
802
810 Excepciones:
803 Excepciones:
811 Si un determinado file no puede ser abierto
804 Si un determinado file no puede ser abierto
812 """
805 """
813
814 nFiles = 0
806 nFiles = 0
815 fileOk_flag = False
807 fileOk_flag = False
816 firstTime_flag = True
808 firstTime_flag = True
@@ -869,48 +861,27 class JRODataReader(JRODataIO):
869 if self.fp != None: self.fp.close()
861 if self.fp != None: self.fp.close()
870 self.fp = open(fullfilename, 'rb')
862 self.fp = open(fullfilename, 'rb')
871 self.flagNoMoreFiles = 0
863 self.flagNoMoreFiles = 0
872 # print '[Reading] Setting the file: %s' % fullfilename
864 # print '[Reading] Setting the file: %s' % fullfilename
873 else:
865 else:
874 self.fileSize = 0
866 self.fileSize = 0
875 self.filename = None
867 self.filename = None
876 self.flagIsNewFile = 0
868 self.flagIsNewFile = 0
877 self.fp = None
869 self.fp = None
878 self.flagNoMoreFiles = 1
870 self.flagNoMoreFiles = 1
879 # print '[Reading] No more files to read'
871 # print '[Reading] No more files to read'
880
872
881 return fileOk_flag
873 return fileOk_flag
882
874
883 def setNextFile(self):
875 def setNextFile(self):
884 if self.fp != None:
876 if self.fp != None:
885 self.fp.close()
877 self.fp.close()
878
886 if self.online:
879 if self.online:
887 newFile = self.__setNextFileOnline()
880 newFile = self.__setNextFileOnline()
888 else:
881 else:
889 newFile = self.__setNextFileOffline()
882 newFile = self.__setNextFileOffline()
883
890 if not(newFile):
884 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
914 print '[Reading] No more files to read'
885 print '[Reading] No more files to read'
915 return 0
886 return 0
916
887
@@ -951,7 +922,7 class JRODataReader(JRODataIO):
951 return 1
922 return 1
952
923
953 if self.fileSize == self.fileSizeByHeader:
924 if self.fileSize == self.fileSizeByHeader:
954 # self.flagEoF = True
925 # self.flagEoF = True
955 return 0
926 return 0
956
927
957 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
928 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
@@ -1005,13 +976,13 class JRODataReader(JRODataIO):
1005 else:
976 else:
1006 self.fp.seek(self.fp.tell() - neededsize)
977 self.fp.seek(self.fp.tell() - neededsize)
1007 break
978 break
1008
979
1009 # csize = self.fileSize - self.fp.tell()
980 # csize = self.fileSize - self.fp.tell()
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
981 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 # factor = int(csize/neededsize)
982 # factor = int(csize/neededsize)
1012 # if factor > 0:
983 # if factor > 0:
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
984 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014
985
1015 self.flagIsNewFile = 0
986 self.flagIsNewFile = 0
1016 self.__isFirstTimeOnline = 0
987 self.__isFirstTimeOnline = 0
1017
988
@@ -1019,10 +990,10 class JRODataReader(JRODataIO):
1019 #if self.server is None:
990 #if self.server is None:
1020 if self.fp == None:
991 if self.fp == None:
1021 return 0
992 return 0
1022
993
1023 # if self.online:
994 # if self.online:
1024 # self.__jumpToLastBlock()
995 # self.__jumpToLastBlock()
1025
996
1026 if self.flagIsNewFile:
997 if self.flagIsNewFile:
1027 self.lastUTTime = self.basicHeaderObj.utc
998 self.lastUTTime = self.basicHeaderObj.utc
1028 return 1
999 return 1
@@ -1065,12 +1036,14 class JRODataReader(JRODataIO):
1065
1036
1066 #Skip block out of startTime and endTime
1037 #Skip block out of startTime and endTime
1067 while True:
1038 while True:
1068 if not(self.__setNewBlock()):
1039 if not(self.__setNewBlock()):
1069 print 'returning'
1070 return 0
1040 return 0
1041
1071 if not(self.readBlock()):
1042 if not(self.readBlock()):
1072 return 0
1043 return 0
1044
1073 self.getBasicHeader()
1045 self.getBasicHeader()
1046
1074 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075
1048
1076 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
@@ -1082,8 +1055,8 class JRODataReader(JRODataIO):
1082
1055
1083 if self.verbose:
1056 if self.verbose:
1084 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1057 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 self.processingHeaderObj.dataBlocksPerFile,
1058 self.processingHeaderObj.dataBlocksPerFile,
1086 self.dataOut.datatime.ctime())
1059 self.dataOut.datatime.ctime())
1087 return 1
1060 return 1
1088
1061
1089 def __readFirstHeader(self):
1062 def __readFirstHeader(self):
@@ -1114,8 +1087,8 class JRODataReader(JRODataIO):
1114 self.dtype = datatype_str
1087 self.dtype = datatype_str
1115 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1088 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1089 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1090 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1091 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 self.getBlockDimension()
1092 self.getBlockDimension()
1120
1093
1121 def __verifyFile(self, filename, msgFlag=True):
1094 def __verifyFile(self, filename, msgFlag=True):
@@ -1277,13 +1250,13 class JRODataReader(JRODataIO):
1277
1250
1278 def setup(self,
1251 def setup(self,
1279 path=None,
1252 path=None,
1280 startDate=None,
1253 startDate=None,
1281 endDate=None,
1254 endDate=None,
1282 startTime=datetime.time(0,0,0),
1255 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1256 endTime=datetime.time(23,59,59),
1284 set=None,
1257 set=None,
1285 expLabel = "",
1258 expLabel = "",
1286 ext = None,
1259 ext = None,
1287 online = False,
1260 online = False,
1288 delay = 60,
1261 delay = 60,
1289 walk = True,
1262 walk = True,
@@ -1292,139 +1265,108 class JRODataReader(JRODataIO):
1292 realtime=False,
1265 realtime=False,
1293 blocksize=None,
1266 blocksize=None,
1294 blocktime=None,
1267 blocktime=None,
1268 skip=None,
1269 cursor=None,
1270 warnings=True,
1295 verbose=True,
1271 verbose=True,
1296 **kwargs):
1272 server=None):
1273 if server is not None:
1274 if 'tcp://' in server:
1275 address = server
1276 else:
1277 address = 'ipc:///tmp/%s' % server
1278 self.server = address
1279 self.context = zmq.Context()
1280 self.receiver = self.context.socket(zmq.PULL)
1281 self.receiver.connect(self.server)
1282 time.sleep(0.5)
1283 print '[Starting] ReceiverData from {}'.format(self.server)
1284 else:
1285 self.server = None
1286 if path == None:
1287 raise ValueError, "[Reading] The path is not valid"
1297
1288
1298 if path == None:
1289 if ext == None:
1299 raise ValueError, "[Reading] The path is not valid"
1290 ext = self.ext
1300
1301
1291
1302 if ext == None:
1292 if online:
1303 ext = self.ext
1293 print "[Reading] Searching files in online mode..."
1304
1294
1305 self.verbose=verbose
1295 for nTries in range( self.nTries ):
1306 self.path = path
1296 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1307 self.startDate = startDate
1297
1308 self.endDate = endDate
1298 if fullpath:
1309 self.startTime = startTime
1299 break
1310 self.endTime = endTime
1300
1311 self.set = set
1301 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1312 self.expLabel = expLabel
1302 sleep( self.delay )
1313 self.ext = ext
1303
1314 self.online = online
1304 if not(fullpath):
1315 self.delay = delay
1305 print "[Reading] There 'isn't any valid file in %s" % path
1316 self.walk = walk
1306 return
1317 self.getblock = getblock
1307
1318 self.nTxs = nTxs
1308 self.year = year
1319 self.realtime = realtime
1309 self.doy = doy
1320 self.blocksize = blocksize
1310 self.set = set - 1
1321 self.blocktime = blocktime
1311 self.path = path
1322
1312 self.foldercounter = foldercounter
1323
1313 last_set = None
1324 if self.firstTime is True:
1314 else:
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1315 print "[Reading] Searching files in offline mode ..."
1316 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 startTime=startTime, endTime=endTime,
1317 startTime=startTime, endTime=endTime,
1327 set=set, expLabel=expLabel, ext=ext,
1318 set=set, expLabel=expLabel, ext=ext,
1328 walk=walk)
1319 walk=walk, cursor=cursor,
1329 if filenameList is not None: filenameList = filenameList[:-1]
1320 skip=skip)
1330
1321
1331 if pathList is not None and filenameList is not None and online:
1322 if not(pathList):
1332 self.onlineWithDate = True
1333 online = False
1334 self.fileIndex = -1
1323 self.fileIndex = -1
1335 self.pathList = pathList
1324 self.pathList = []
1336 self.filenameList = filenameList
1325 self.filenameList = []
1337 file_name = os.path.basename(filenameList[-1])
1326 return
1338 basename, ext = os.path.splitext(file_name)
1327
1339 last_set = int(basename[-3:])
1340
1341 if online:
1342 print "[Reading] Searching files in online mode..."
1343
1344 for nTries in range(self.nTries):
1345 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1346 expLabel=expLabel,
1347 ext=ext,
1348 walk=walk,
1349 startDate=startDate,
1350 startTime=startTime,
1351 set=set)
1352
1353 if fullpath:
1354 break
1355 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1356 sleep( self.delay )
1357
1358 if not(fullpath):
1359 print "[Reading] There 'isn't any valid file in %s" % path
1360 return
1361
1362 self.year = year
1363 self.doy = doy
1364 self.set = set - 1
1365 self.path = path
1366 self.foldercounter = foldercounter
1367 last_set = None
1368 else:
1369 print "[Reading] Searching files in offline mode ..."
1370 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1371 startTime=startTime, endTime=endTime,
1372 set=set, expLabel=expLabel, ext=ext,
1373 walk=walk)
1374
1375 if not(pathList):
1376 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1377 # datetime.datetime.combine(startDate,startTime).ctime(),
1378 # datetime.datetime.combine(endDate,endTime).ctime())
1379
1380 # sys.exit(-1)
1381
1382 self.fileIndex = -1
1383 self.pathList = []
1384 self.filenameList = []
1385 return
1386
1387 self.fileIndex = -1
1388 self.pathList = pathList
1389 self.filenameList = filenameList
1390 file_name = os.path.basename(filenameList[-1])
1391 basename, ext = os.path.splitext(file_name)
1392 last_set = int(basename[-3:])
1393
1394
1395 self.online = online
1396 self.realtime = realtime
1397 self.delay = delay
1398 ext = ext.lower()
1399 self.ext = ext
1400 self.getByBlock = getblock
1401 self.nTxs = nTxs
1402 self.startTime = startTime
1403 self.endTime = endTime
1404
1405
1406 #Added-----------------
1407 self.selBlocksize = blocksize
1408 self.selBlocktime = blocktime
1409
1410
1411 if not(self.setNextFile()):
1412 if (startDate!=None) and (endDate!=None):
1413 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1414 elif startDate != None:
1415 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1416 else:
1417 print "[Reading] No files"
1418
1419 self.fileIndex = -1
1328 self.fileIndex = -1
1420 self.pathList = []
1329 self.pathList = pathList
1421 self.filenameList = []
1330 self.filenameList = filenameList
1422 return
1331 file_name = os.path.basename(filenameList[-1])
1332 basename, ext = os.path.splitext(file_name)
1333 last_set = int(basename[-3:])
1334
1335 self.online = online
1336 self.realtime = realtime
1337 self.delay = delay
1338 ext = ext.lower()
1339 self.ext = ext
1340 self.getByBlock = getblock
1341 self.nTxs = nTxs
1342 self.startTime = startTime
1343 self.endTime = endTime
1344
1345 #Added-----------------
1346 self.selBlocksize = blocksize
1347 self.selBlocktime = blocktime
1348
1349 # Verbose-----------
1350 self.verbose = verbose
1351 self.warnings = warnings
1352
1353 if not(self.setNextFile()):
1354 if (startDate!=None) and (endDate!=None):
1355 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1356 elif startDate != None:
1357 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1358 else:
1359 print "[Reading] No files"
1360
1361 self.fileIndex = -1
1362 self.pathList = []
1363 self.filenameList = []
1364 return
1423
1365
1424 # self.getBasicHeader()
1366 # self.getBasicHeader()
1425
1367
1426 if last_set != None:
1368 if last_set != None:
1427 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1369 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1428 return
1370 return
1429
1371
1430 def getBasicHeader(self):
1372 def getBasicHeader(self):
@@ -1442,10 +1384,10 class JRODataReader(JRODataIO):
1442 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1384 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1443
1385
1444 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1386 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1445
1387
1446 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1388 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1447
1389
1448
1390
1449 def getFirstHeader(self):
1391 def getFirstHeader(self):
1450
1392
1451 raise NotImplementedError
1393 raise NotImplementedError
@@ -1494,53 +1436,50 class JRODataReader(JRODataIO):
1494 self.__printInfo = False
1436 self.__printInfo = False
1495
1437
1496 def run(self,
1438 def run(self,
1497 path=None,
1439 path=None,
1498 startDate=None,
1440 startDate=None,
1499 endDate=None,
1441 endDate=None,
1500 startTime=datetime.time(0,0,0),
1442 startTime=datetime.time(0,0,0),
1501 endTime=datetime.time(23,59,59),
1443 endTime=datetime.time(23,59,59),
1502 set=None,
1444 set=None,
1503 expLabel = "",
1445 expLabel = "",
1504 ext = None,
1446 ext = None,
1505 online = False,
1447 online = False,
1506 delay = 60,
1448 delay = 60,
1507 walk = True,
1449 walk = True,
1508 getblock = False,
1450 getblock = False,
1509 nTxs = 1,
1451 nTxs = 1,
1510 realtime=False,
1452 realtime=False,
1511 blocksize=None,
1453 blocksize=None,
1512 blocktime=None,
1454 blocktime=None,
1513 queue=None,
1455 skip=None,
1514 skip=None,
1456 cursor=None,
1515 cursor=None,
1457 warnings=True,
1516 warnings=True,
1458 server=None,
1517 server=None,
1459 verbose=True, **kwargs):
1518 verbose=True, **kwargs):
1519
1460
1520 if not(self.isConfig):
1461 if not(self.isConfig):
1521 # self.dataOut = dataOut
1462 self.setup(path=path,
1522 self.setup( path=path,
1463 startDate=startDate,
1523 startDate=startDate,
1464 endDate=endDate,
1524 endDate=endDate,
1465 startTime=startTime,
1525 startTime=startTime,
1466 endTime=endTime,
1526 endTime=endTime,
1467 set=set,
1527 set=set,
1468 expLabel=expLabel,
1528 expLabel=expLabel,
1469 ext=ext,
1529 ext=ext,
1470 online=online,
1530 online=online,
1471 delay=delay,
1531 delay=delay,
1472 walk=walk,
1532 walk=walk,
1473 getblock=getblock,
1533 getblock=getblock,
1474 nTxs=nTxs,
1534 nTxs=nTxs,
1475 realtime=realtime,
1535 realtime=realtime,
1476 blocksize=blocksize,
1536 blocksize=blocksize,
1477 blocktime=blocktime,
1537 blocktime=blocktime,
1478 skip=skip,
1538 queue=queue,
1479 cursor=cursor,
1539 skip=skip,
1480 warnings=warnings,
1540 cursor=cursor,
1481 server=server,
1541 warnings=warnings,
1482 verbose=verbose)
1542 server=server,
1543 verbose=verbose, **kwargs)
1544 self.isConfig = True
1483 self.isConfig = True
1545 if server is None:
1484 if server is None:
1546 self.getData()
1485 self.getData()
@@ -1663,9 +1602,9 class JRODataWriter(JRODataIO):
1663 Return:
1602 Return:
1664 None
1603 None
1665 """
1604 """
1666
1605
1667 # CALCULAR PARAMETROS
1606 # CALCULAR PARAMETROS
1668
1607
1669 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1608 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1670 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1609 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1671
1610
@@ -1792,7 +1731,7 class JRODataWriter(JRODataIO):
1792
1731
1793 return 1
1732 return 1
1794
1733
1795 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True):
1734 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1796 """
1735 """
1797 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1736 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1798
1737
Show file before | Show file after | Hide comments
@@ -453,7 +453,7 class FitsReader(ProcessingUnit):
453 # self.blockIndex = 1
453 # self.blockIndex = 1
454 return 1
454 return 1
455
455
456 def __searchFilesOffLine(self,
456 def searchFilesOffLine(self,
457 path,
457 path,
458 startDate,
458 startDate,
459 endDate,
459 endDate,
@@ -559,7 +559,7 class FitsReader(ProcessingUnit):
559
559
560 if not(online):
560 if not(online):
561 print "Searching files in offline mode ..."
561 print "Searching files in offline mode ..."
562 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
562 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
563 startTime=startTime, endTime=endTime,
563 startTime=startTime, endTime=endTime,
564 set=set, expLabel=expLabel, ext=ext,
564 set=set, expLabel=expLabel, ext=ext,
565 walk=walk)
565 walk=walk)
Show file before | Show file after | Hide comments
@@ -415,7 +415,7 class HFReader(ProcessingUnit):
415
415
416
416
417
417
418 def __searchFilesOffline(self,
418 def searchFilesOffLine(self,
419 path,
419 path,
420 startDate,
420 startDate,
421 endDate,
421 endDate,
@@ -438,7 +438,7 class HFReader(ProcessingUnit):
438
438
439 return
439 return
440
440
441 def __searchFilesOnline(self,
441 def searchFilesOnLine(self,
442 path,
442 path,
443 expLabel= "",
443 expLabel= "",
444 ext=None,
444 ext=None,
@@ -636,10 +636,10 class HFReader(ProcessingUnit):
636 if not(online):
636 if not(online):
637 print "Searching files in offline mode..."
637 print "Searching files in offline mode..."
638
638
639 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 else:
640 else:
641 print "Searching files in online mode..."
641 print "Searching files in online mode..."
642 self.__searchFilesOnline(path, walk,ext,set=set)
642 self.searchFilesOnLine(path, walk,ext,set=set)
643 if set==None:
643 if set==None:
644 pass
644 pass
645 else:
645 else:
@@ -647,7 +647,7 class HFReader(ProcessingUnit):
647
647
648 # for nTries in range(self.nTries):
648 # for nTries in range(self.nTries):
649 #
649 #
650 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 #
651 #
652 # if fullpath:
652 # if fullpath:
653 # break
653 # break
Show file before | Show file after | Hide comments
@@ -106,9 +106,9 class AMISRReader(ProcessingUnit):
106 #self.findFiles()
106 #self.findFiles()
107 if not(online):
107 if not(online):
108 #Busqueda de archivos offline
108 #Busqueda de archivos offline
109 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
109 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
110 else:
110 else:
111 self.__searchFilesOnline(path, startDate, endDate, startTime,endTime,walk)
111 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
112
112
113 if not(self.filenameList):
113 if not(self.filenameList):
114 print "There is no files into the folder: %s"%(path)
114 print "There is no files into the folder: %s"%(path)
@@ -329,7 +329,7 class AMISRReader(ProcessingUnit):
329 self.dirnameList = new_dirnameList
329 self.dirnameList = new_dirnameList
330 return 1
330 return 1
331
331
332 def __searchFilesOnline(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 endTime=datetime.time(23,59,59),walk=True):
333 endTime=datetime.time(23,59,59),walk=True):
334
334
335 if endDate ==None:
335 if endDate ==None:
@@ -349,7 +349,7 class AMISRReader(ProcessingUnit):
349 return
349 return
350
350
351
351
352 def __searchFilesOffline(self,
352 def searchFilesOffLine(self,
353 path,
353 path,
354 startDate,
354 startDate,
355 endDate,
355 endDate,
Show file before | Show file after | Hide comments
@@ -97,7 +97,7 class ParamReader(ProcessingUnit):
97 self.timezone = 'lt'
97 self.timezone = 'lt'
98
98
99 print "[Reading] Searching files in offline mode ..."
99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 startTime=startTime, endTime=endTime,
101 startTime=startTime, endTime=endTime,
102 ext=ext, walk=walk)
102 ext=ext, walk=walk)
103
103
@@ -115,7 +115,7 class ParamReader(ProcessingUnit):
115
115
116 return
116 return
117
117
118 def __searchFilesOffLine(self,
118 def searchFilesOffLine(self,
119 path,
119 path,
120 startDate=None,
120 startDate=None,
121 endDate=None,
121 endDate=None,
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now