##// END OF EJS Templates
schain
RSS
Change multiSchain by MPProject
Juan C. Espinoza -
r1052:3fb7c359028f
parent child
Show More
Side by Side Unified
Context file:
r1052:3fb7c359028f Loading diff...:
Show file before | Show file after | Hide comments
@@ -9,56 +9,53 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 from multiprocessing import Process, Queue, cpu_count
13
14 import schainpy
15 import schainpy.admin
16 from schainpy.utils.log import logToFile
12 from multiprocessing import Process, cpu_count
17 13
18 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 15 from xml.dom import minidom
20 16
17 import schainpy
18 import schainpy.admin
21 19 from schainpy.model import *
22 from time import sleep
20 from schainpy.utils import log
23 21
22 DTYPES = {
23 'Voltage': '.r',
24 'Spectra': '.pdata'
25 }
24 26
27 def MPProject(project, n=cpu_count()):
28 '''
29 Project wrapper to run schain in n processes
30 '''
25 31
26 def prettify(elem):
27 """Return a pretty-printed XML string for the Element.
28 """
29 rough_string = tostring(elem, 'utf-8')
30 reparsed = minidom.parseString(rough_string)
31 return reparsed.toprettyxml(indent=" ")
32
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 skip = 0
35 cursor = 0
36 nFiles = None
37 processes = []
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
32 rconf = project.getReadUnitObj()
33 op = rconf.getOperationObj('run')
34 dt1 = op.getParameterValue('startDate')
35 dt2 = op.getParameterValue('endDate')
40 36 days = (dt2 - dt1).days
41 37
42 38 for day in range(days+1):
43 39 skip = 0
44 40 cursor = 0
45 q = Queue()
46 41 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
49 firstProcess.start()
50 if by_day:
51 continue
52 nFiles = q.get()
42 dt = dt1 + datetime.timedelta(day)
43 dt_str = dt.strftime('%Y/%m/%d')
44 reader = JRODataReader()
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 startDate=dt,
47 endDate=dt,
48 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
53 50 if nFiles==0:
54 51 continue
55 firstProcess.terminate()
56 skip = int(math.ceil(nFiles/nProcess))
57 while True:
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
59 processes[cursor].start()
60 if nFiles < cursor*skip:
61 break
52 skip = int(math.ceil(nFiles/n))
53 while nFiles > cursor*skip:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 skip=skip)
56 p = project.clone()
57 p.start()
58 processes.append(p)
62 59 cursor += 1
63 60
64 61 def beforeExit(exctype, value, trace):
@@ -75,7 +72,6 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_da
75 72
76 73 time.sleep(3)
77 74
78
79 75 class ParameterConf():
80 76
81 77 id = None
@@ -112,7 +108,7 class ParameterConf():
112 108 return self.__formated_value
113 109
114 110 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
111 raise ValueError, '%s: This parameter value is empty' %self.name
116 112
117 113 if format == 'list':
118 114 strList = value.split(',')
@@ -122,10 +118,10 class ParameterConf():
122 118 return self.__formated_value
123 119
124 120 if format == 'intlist':
125 """
121 '''
126 122 Example:
127 123 value = (0,1,2)
128 """
124 '''
129 125
130 126 new_value = ast.literal_eval(value)
131 127
@@ -137,10 +133,10 class ParameterConf():
137 133 return self.__formated_value
138 134
139 135 if format == 'floatlist':
140 """
136 '''
141 137 Example:
142 138 value = (0.5, 1.4, 2.7)
143 """
139 '''
144 140
145 141 new_value = ast.literal_eval(value)
146 142
@@ -170,38 +166,38 class ParameterConf():
170 166 return self.__formated_value
171 167
172 168 if format == 'pairslist':
173 """
169 '''
174 170 Example:
175 171 value = (0,1),(1,2)
176 """
172 '''
177 173
178 174 new_value = ast.literal_eval(value)
179 175
180 176 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
182 178
183 179 if type(new_value[0]) not in (tuple, list):
184 180 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
186 182 new_value = [new_value]
187 183
188 184 for thisPair in new_value:
189 185 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
191 187
192 188 self.__formated_value = new_value
193 189
194 190 return self.__formated_value
195 191
196 192 if format == 'multilist':
197 """
193 '''
198 194 Example:
199 195 value = (0,1,2),(3,4,5)
200 """
196 '''
201 197 multiList = ast.literal_eval(value)
202 198
203 199 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
200 multiList = ast.literal_eval('(' + value + ')')
205 201
206 202 self.__formated_value = multiList
207 203
@@ -263,7 +259,7 class ParameterConf():
263 259
264 260 def printattr(self):
265 261
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
267 263
268 264 class OperationConf():
269 265
@@ -372,6 +368,8 class OperationConf():
372 368
373 369 def addParameter(self, name, value, format='str'):
374 370
371 if value is None:
372 return None
375 373 id = self.__getNewId()
376 374
377 375 parmConfObj = ParameterConf()
@@ -431,7 +429,7 class OperationConf():
431 429
432 430 def printattr(self):
433 431
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
435 433 self.id,
436 434 self.name,
437 435 self.type,
@@ -444,12 +442,11 class OperationConf():
444 442
445 443
446 444 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
445 raise ValueError, 'This operation type cannot be created'
448 446
449 447 if self.type == 'plotter':
450 #Plotter(plotter_name)
451 448 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453 450
454 451 opObj = Plotter(self.name, plotter_queue)
455 452
@@ -564,7 +561,7 class ProcUnitConf():
564 561
565 562 #Compatible with old signal chain version
566 563 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
564 raise ValueError, 'datatype or name should be defined'
568 565
569 566 if name==None:
570 567 if 'Proc' in datatype:
@@ -595,7 +592,7 class ProcUnitConf():
595 592
596 593 def addParameter(self, **kwargs):
597 594 '''
598 Add parameters to "run" operation
595 Add parameters to 'run' operation
599 596 '''
600 597 opObj = self.opConfObjList[0]
601 598
@@ -633,11 +630,11 class ProcUnitConf():
633 630 self.datatype = upElement.get('datatype')
634 631 self.inputId = upElement.get('inputId')
635 632
636 if self.ELEMENTNAME == "ReadUnit":
637 self.datatype = self.datatype.replace("Reader", "")
633 if self.ELEMENTNAME == 'ReadUnit':
634 self.datatype = self.datatype.replace('Reader', '')
638 635
639 if self.ELEMENTNAME == "ProcUnit":
640 self.datatype = self.datatype.replace("Proc", "")
636 if self.ELEMENTNAME == 'ProcUnit':
637 self.datatype = self.datatype.replace('Proc', '')
641 638
642 639 if self.inputId == 'None':
643 640 self.inputId = '0'
@@ -653,7 +650,7 class ProcUnitConf():
653 650
654 651 def printattr(self):
655 652
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
657 654 self.id,
658 655 self.name,
659 656 self.datatype,
@@ -707,18 +704,10 class ProcUnitConf():
707 704
708 705 kwargs[parmConfObj.name] = parmConfObj.getValue()
709 706
710 #ini = time.time()
711
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
713 707 sts = self.procUnitObj.call(opType = opConfObj.type,
714 708 opName = opConfObj.name,
715 709 opId = opConfObj.id)
716 710
717 # total_time = time.time() - ini
718 #
719 # if total_time > 0.002:
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
721
722 711 is_ok = is_ok or sts
723 712
724 713 return is_ok
@@ -762,11 +751,12 class ReadUnitConf(ProcUnitConf):
762 751
763 752 return self.ELEMENTNAME
764 753
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
767 757 #Compatible with old signal chain version
768 758 if datatype==None and name==None:
769 raise ValueError, "datatype or name should be defined"
759 raise ValueError, 'datatype or name should be defined'
770 760
771 761 if name==None:
772 762 if 'Reader' in datatype:
@@ -785,39 +775,28 class ReadUnitConf(ProcUnitConf):
785 775 self.endDate = endDate
786 776 self.startTime = startTime
787 777 self.endTime = endTime
788
789 778 self.inputId = '0'
790 779 self.parentId = parentId
791 self.queue = queue
792 780 self.server = server
793 781 self.addRunOperation(**kwargs)
794 782
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
796
797 #Compatible with old signal chain version
798 if datatype==None and name==None:
799 raise ValueError, "datatype or name should be defined"
783 def update(self, **kwargs):
800 784
801 if name==None:
785 if 'datatype' in kwargs:
786 datatype = kwargs.pop('datatype')
802 787 if 'Reader' in datatype:
803 name = datatype
788 self.name = datatype
804 789 else:
805 name = '%sReader' %(datatype)
790 self.name = '%sReader' %(datatype)
791 self.datatype = self.name.replace('Reader', '')
806 792
807 if datatype==None:
808 datatype = name.replace('Reader','')
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
809 794
810 self.datatype = datatype
811 self.name = name
812 self.path = path
813 self.startDate = startDate
814 self.endDate = endDate
815 self.startTime = startTime
816 self.endTime = endTime
795 for attr in attrs:
796 if attr in kwargs:
797 setattr(self, attr, kwargs.pop(attr))
817 798
818 799 self.inputId = '0'
819 self.parentId = parentId
820
821 800 self.updateRunOperation(**kwargs)
822 801
823 802 def removeOperations(self):
@@ -838,7 +817,7 class ReadUnitConf(ProcUnitConf):
838 817 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
839 818 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
840 819 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
820
842 821 for key, value in kwargs.items():
843 822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
844 823 else:
@@ -864,17 +843,6 class ReadUnitConf(ProcUnitConf):
864 843
865 844 return opObj
866 845
867 # def makeXml(self, projectElement):
868 #
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
870 # procUnitElement.set('id', str(self.id))
871 # procUnitElement.set('name', self.name)
872 # procUnitElement.set('datatype', self.datatype)
873 # procUnitElement.set('inputId', str(self.inputId))
874 #
875 # for opConfObj in self.opConfObjList:
876 # opConfObj.makeXml(procUnitElement)
877
878 846 def readXml(self, upElement):
879 847
880 848 self.id = upElement.get('id')
@@ -882,8 +850,8 class ReadUnitConf(ProcUnitConf):
882 850 self.datatype = upElement.get('datatype')
883 851 self.inputId = upElement.get('inputId')
884 852
885 if self.ELEMENTNAME == "ReadUnit":
886 self.datatype = self.datatype.replace("Reader", "")
853 if self.ELEMENTNAME == 'ReadUnit':
854 self.datatype = self.datatype.replace('Reader', '')
887 855
888 856 if self.inputId == 'None':
889 857 self.inputId = '0'
@@ -905,8 +873,9 class ReadUnitConf(ProcUnitConf):
905 873 self.endTime = opConfObj.getParameterValue('endTime')
906 874
907 875 class Project(Process):
876
908 877 id = None
909 name = None
878 # name = None
910 879 description = None
911 880 filename = None
912 881
@@ -916,13 +885,13 class Project(Process):
916 885
917 886 plotterQueue = None
918 887
919 def __init__(self, plotter_queue=None, logfile=None):
888 def __init__(self, plotter_queue=None):
889
920 890 Process.__init__(self)
921 891 self.id = None
922 self.name = None
892 # self.name = None
923 893 self.description = None
924 if logfile is not None:
925 logToFile(logfile)
894
926 895 self.plotterQueue = plotter_queue
927 896
928 897 self.procUnitConfObjDict = {}
@@ -972,18 +941,28 class Project(Process):
972 941
973 942 self.procUnitConfObjDict = newProcUnitConfObjDict
974 943
975 def setup(self, id, name, description):
944 def setup(self, id, name='', description=''):
976 945
946 print
947 print '*'*60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
950 print
977 951 self.id = str(id)
978 self.name = name
979 952 self.description = description
980 953
981 954 def update(self, name, description):
982 955
983 self.name = name
984 956 self.description = description
985 957
958 def clone(self):
959
960 p = Project()
961 p.procUnitConfObjDict = self.procUnitConfObjDict
962 return p
963
986 964 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965
987 966 if id is None:
988 967 idReadUnit = self.__getNewId()
989 968 else:
@@ -1021,7 +1000,7 class Project(Process):
1021 1000 def getReadUnitObj(self):
1022 1001
1023 1002 for obj in self.procUnitConfObjDict.values():
1024 if obj.getElementName() == "ReadUnit":
1003 if obj.getElementName() == 'ReadUnit':
1025 1004 return obj
1026 1005
1027 1006 return None
@@ -1066,20 +1045,20 class Project(Process):
1066 1045 if self.filename:
1067 1046 filename = self.filename
1068 1047 else:
1069 filename = "schain.xml"
1048 filename = 'schain.xml'
1070 1049
1071 1050 if not filename:
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1051 print 'filename has not been defined. Use setFilename(filename) for do it.'
1073 1052 return 0
1074 1053
1075 1054 abs_file = os.path.abspath(filename)
1076 1055
1077 1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1079 1058 return 0
1080 1059
1081 1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1082 print "File %s already exists and it could not be overwriten" %abs_file
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1083 1062 return 0
1084 1063
1085 1064 self.makeXml()
@@ -1093,13 +1072,13 class Project(Process):
1093 1072 def readXml(self, filename = None):
1094 1073
1095 1074 if not filename:
1096 print "filename is not defined"
1075 print 'filename is not defined'
1097 1076 return 0
1098 1077
1099 1078 abs_file = os.path.abspath(filename)
1100 1079
1101 1080 if not os.path.isfile(abs_file):
1102 print "%s file does not exist" %abs_file
1081 print '%s file does not exist' %abs_file
1103 1082 return 0
1104 1083
1105 1084 self.projectElement = None
@@ -1108,7 +1087,7 class Project(Process):
1108 1087 try:
1109 1088 self.projectElement = ElementTree().parse(abs_file)
1110 1089 except:
1111 print "Error reading %s, verify file format" %filename
1090 print 'Error reading %s, verify file format' %filename
1112 1091 return 0
1113 1092
1114 1093 self.project = self.projectElement.tag
@@ -1145,7 +1124,7 class Project(Process):
1145 1124
1146 1125 def printattr(self):
1147 1126
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1149 1128 self.name,
1150 1129 self.description)
1151 1130
@@ -1179,7 +1158,7 class Project(Process):
1179 1158
1180 1159 self.__connect(puObjIN, thisPUObj)
1181 1160
1182 def __handleError(self, procUnitConfObj, send_email=True):
1161 def __handleError(self, procUnitConfObj, send_email=False):
1183 1162
1184 1163 import socket
1185 1164
@@ -1187,33 +1166,33 class Project(Process):
1187 1166 sys.exc_info()[1],
1188 1167 sys.exc_info()[2])
1189 1168
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1191 print "***** %s" %err[-1]
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1170 print '***** %s' %err[-1]
1192 1171
1193 message = "".join(err)
1172 message = ''.join(err)
1194 1173
1195 1174 sys.stderr.write(message)
1196 1175
1197 1176 if not send_email:
1198 1177 return
1199 1178
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1201 1180
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1205 subtitle += "Configuration file: %s\n" %self.filename
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1184 subtitle += 'Configuration file: %s\n' %self.filename
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1207 1186
1208 1187 readUnitConfObj = self.getReadUnitObj()
1209 1188 if readUnitConfObj:
1210 subtitle += "\nInput parameters:\n"
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1189 subtitle += '\nInput parameters:\n'
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1217 1196
1218 1197 adminObj = schainpy.admin.SchainNotify()
1219 1198 adminObj.sendAlert(message=message,
@@ -1228,15 +1207,15 class Project(Process):
1228 1207 return 0
1229 1208
1230 1209 def runController(self):
1231 """
1210 '''
1232 1211 returns 0 when this process has been stopped, 1 otherwise
1233 """
1212 '''
1234 1213
1235 1214 if self.isPaused():
1236 print "Process suspended"
1215 print 'Process suspended'
1237 1216
1238 1217 while True:
1239 sleep(0.1)
1218 time.sleep(0.1)
1240 1219
1241 1220 if not self.isPaused():
1242 1221 break
@@ -1244,10 +1223,10 class Project(Process):
1244 1223 if self.isStopped():
1245 1224 break
1246 1225
1247 print "Process reinitialized"
1226 print 'Process reinitialized'
1248 1227
1249 1228 if self.isStopped():
1250 print "Process stopped"
1229 print 'Process stopped'
1251 1230 return 0
1252 1231
1253 1232 return 1
@@ -1258,29 +1237,23 class Project(Process):
1258 1237
1259 1238 def setPlotterQueue(self, plotter_queue):
1260 1239
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1240 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1262 1241
1263 1242 def getPlotterQueue(self):
1264 1243
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1244 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1266 1245
1267 1246 def useExternalPlotter(self):
1268 1247
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1248 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270 1249
1250 def run(self):
1271 1251
1272 def run(self, filename=None):
1252 log.success('Starting {}'.format(self.name))
1273 1253
1274 # self.writeXml(filename)
1275 1254 self.createObjects()
1276 1255 self.connectObjects()
1277 1256
1278 print
1279 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1282 print
1283
1284 1257 keyList = self.procUnitConfObjDict.keys()
1285 1258 keyList.sort()
1286 1259
@@ -1289,7 +1262,6 class Project(Process):
1289 1262 is_ok = False
1290 1263
1291 1264 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293 1265
1294 1266 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 1267
@@ -1300,19 +1272,18 class Project(Process):
1300 1272 is_ok = False
1301 1273 break
1302 1274 except ValueError, e:
1303 sleep(0.5)
1275 time.sleep(0.5)
1304 1276 self.__handleError(procUnitConfObj, send_email=True)
1305 1277 is_ok = False
1306 1278 break
1307 1279 except:
1308 sleep(0.5)
1280 time.sleep(0.5)
1309 1281 self.__handleError(procUnitConfObj)
1310 1282 is_ok = False
1311 1283 break
1312 1284
1313 1285 #If every process unit finished so end process
1314 1286 if not(is_ok):
1315 # print "Every process unit have finished"
1316 1287 break
1317 1288
1318 1289 if not self.runController():
@@ -1322,3 +1293,5 class Project(Process):
1322 1293 for procKey in keyList:
1323 1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 1295 procUnitConfObj.close()
1296
1297 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
@@ -6218,3 +6218,6 class ShowMeConsole(QtCore.QObject):
6218 6218 text = text[:-1]
6219 6219
6220 6220 self.textWritten.emit(str(text))
6221
6222 def flush(self):
6223 pass
Show file before | Show file after | Hide comments
@@ -267,7 +267,7 class AMISRReader(ProcessingUnit):
267 267 self.dirnameList = new_dirnameList
268 268 return 1
269 269
270 def __searchFilesOnline(self,
270 def searchFilesOnLine(self,
271 271 path,
272 272 walk=True):
273 273
@@ -287,7 +287,7 class AMISRReader(ProcessingUnit):
287 287 return
288 288
289 289
290 def __searchFilesOffline(self,
290 def searchFilesOffLine(self,
291 291 path,
292 292 startDate,
293 293 endDate,
@@ -494,9 +494,9 class AMISRReader(ProcessingUnit):
494 494 self.online = online
495 495 if not(online):
496 496 #Busqueda de archivos offline
497 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 498 else:
499 self.__searchFilesOnline(path, walk)
499 self.searchFilesOnLine(path, walk)
500 500
501 501 if not(self.filenameList):
502 502 print "There is no files into the folder: %s"%(path)
Show file before | Show file after | Hide comments
@@ -542,7 +542,6 class JRODataIO:
542 542
543 543 class JRODataReader(JRODataIO):
544 544
545 firstTime = True
546 545 online = 0
547 546
548 547 realtime = 0
@@ -579,7 +578,6 class JRODataReader(JRODataIO):
579 578
580 579 selBlocktime = None
581 580
582 onlineWithDate = False
583 581 def __init__(self):
584 582
585 583 """
@@ -603,7 +601,7 class JRODataReader(JRODataIO):
603 601
604 602 raise NotImplementedError
605 603
606 def __searchFilesOffLine(self,
604 def searchFilesOffLine(self,
607 605 path,
608 606 startDate=None,
609 607 endDate=None,
@@ -612,10 +610,10 class JRODataReader(JRODataIO):
612 610 set=None,
613 611 expLabel='',
614 612 ext='.r',
615 queue=None,
616 613 cursor=None,
617 614 skip=None,
618 615 walk=True):
616
619 617 self.filenameList = []
620 618 self.datetimeList = []
621 619
@@ -624,8 +622,7 class JRODataReader(JRODataIO):
624 622 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 623
626 624 if dateList == []:
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 return None, None
625 return [], []
629 626
630 627 if len(dateList) > 1:
631 628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
@@ -636,7 +633,6 class JRODataReader(JRODataIO):
636 633 datetimeList = []
637 634
638 635 for thisPath in pathList:
639 # thisPath = pathList[pathDict[file]]
640 636
641 637 fileList = glob.glob1(thisPath, "*%s" %ext)
642 638 fileList.sort()
@@ -644,10 +640,8 class JRODataReader(JRODataIO):
644 640 skippedFileList = []
645 641
646 642 if cursor is not None and skip is not None:
647 # if cursor*skip > len(fileList):
643
648 644 if skip == 0:
649 if queue is not None:
650 queue.put(len(fileList))
651 645 skippedFileList = []
652 646 else:
653 647 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
@@ -672,19 +666,20 class JRODataReader(JRODataIO):
672 666
673 667 if not(filenameList):
674 668 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 return None, None
669 return [], []
676 670
677 671 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 672 print
679 673
680 for i in range(len(filenameList)):
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
674 # for i in range(len(filenameList)):
675 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 676
683 677 self.filenameList = filenameList
684 678 self.datetimeList = datetimeList
679
685 680 return pathList, filenameList
686 681
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
682 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
688 683
689 684 """
690 685 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
@@ -697,7 +692,7 class JRODataReader(JRODataIO):
697 692
698 693 ext : extension de los files
699 694
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
695 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
701 696
702 697 Return:
703 698 directory : eL directorio donde esta el file encontrado
@@ -708,8 +703,6 class JRODataReader(JRODataIO):
708 703
709 704
710 705 """
711 pathList = None
712 filenameList = None
713 706 if not os.path.isdir(path):
714 707 return None, None, None, None, None, None
715 708
@@ -810,7 +803,6 class JRODataReader(JRODataIO):
810 803 Excepciones:
811 804 Si un determinado file no puede ser abierto
812 805 """
813
814 806 nFiles = 0
815 807 fileOk_flag = False
816 808 firstTime_flag = True
@@ -883,34 +875,13 class JRODataReader(JRODataIO):
883 875 def setNextFile(self):
884 876 if self.fp != None:
885 877 self.fp.close()
878
886 879 if self.online:
887 880 newFile = self.__setNextFileOnline()
888 881 else:
889 882 newFile = self.__setNextFileOffline()
883
890 884 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
914 885 print '[Reading] No more files to read'
915 886 return 0
916 887
@@ -1066,11 +1037,13 class JRODataReader(JRODataIO):
1066 1037 #Skip block out of startTime and endTime
1067 1038 while True:
1068 1039 if not(self.__setNewBlock()):
1069 print 'returning'
1070 1040 return 0
1041
1071 1042 if not(self.readBlock()):
1072 1043 return 0
1044
1073 1045 self.getBasicHeader()
1046
1074 1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075 1048
1076 1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
@@ -1292,66 +1265,39 class JRODataReader(JRODataIO):
1292 1265 realtime=False,
1293 1266 blocksize=None,
1294 1267 blocktime=None,
1268 skip=None,
1269 cursor=None,
1270 warnings=True,
1295 1271 verbose=True,
1296 **kwargs):
1297
1272 server=None):
1273 if server is not None:
1274 if 'tcp://' in server:
1275 address = server
1276 else:
1277 address = 'ipc:///tmp/%s' % server
1278 self.server = address
1279 self.context = zmq.Context()
1280 self.receiver = self.context.socket(zmq.PULL)
1281 self.receiver.connect(self.server)
1282 time.sleep(0.5)
1283 print '[Starting] ReceiverData from {}'.format(self.server)
1284 else:
1285 self.server = None
1298 1286 if path == None:
1299 1287 raise ValueError, "[Reading] The path is not valid"
1300 1288
1301
1302 1289 if ext == None:
1303 1290 ext = self.ext
1304 1291
1305 self.verbose=verbose
1306 self.path = path
1307 self.startDate = startDate
1308 self.endDate = endDate
1309 self.startTime = startTime
1310 self.endTime = endTime
1311 self.set = set
1312 self.expLabel = expLabel
1313 self.ext = ext
1314 self.online = online
1315 self.delay = delay
1316 self.walk = walk
1317 self.getblock = getblock
1318 self.nTxs = nTxs
1319 self.realtime = realtime
1320 self.blocksize = blocksize
1321 self.blocktime = blocktime
1322
1323
1324 if self.firstTime is True:
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 startTime=startTime, endTime=endTime,
1327 set=set, expLabel=expLabel, ext=ext,
1328 walk=walk)
1329 if filenameList is not None: filenameList = filenameList[:-1]
1330
1331 if pathList is not None and filenameList is not None and online:
1332 self.onlineWithDate = True
1333 online = False
1334 self.fileIndex = -1
1335 self.pathList = pathList
1336 self.filenameList = filenameList
1337 file_name = os.path.basename(filenameList[-1])
1338 basename, ext = os.path.splitext(file_name)
1339 last_set = int(basename[-3:])
1340
1341 1292 if online:
1342 1293 print "[Reading] Searching files in online mode..."
1343 1294
1344 1295 for nTries in range(self.nTries):
1345 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1346 expLabel=expLabel,
1347 ext=ext,
1348 walk=walk,
1349 startDate=startDate,
1350 startTime=startTime,
1351 set=set)
1296 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1352 1297
1353 1298 if fullpath:
1354 1299 break
1300
1355 1301 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1356 1302 sleep( self.delay )
1357 1303
@@ -1367,18 +1313,13 class JRODataReader(JRODataIO):
1367 1313 last_set = None
1368 1314 else:
1369 1315 print "[Reading] Searching files in offline mode ..."
1370 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1316 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1371 1317 startTime=startTime, endTime=endTime,
1372 1318 set=set, expLabel=expLabel, ext=ext,
1373 walk=walk)
1319 walk=walk, cursor=cursor,
1320 skip=skip)
1374 1321
1375 1322 if not(pathList):
1376 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1377 # datetime.datetime.combine(startDate,startTime).ctime(),
1378 # datetime.datetime.combine(endDate,endTime).ctime())
1379
1380 # sys.exit(-1)
1381
1382 1323 self.fileIndex = -1
1383 1324 self.pathList = []
1384 1325 self.filenameList = []
@@ -1391,7 +1332,6 class JRODataReader(JRODataIO):
1391 1332 basename, ext = os.path.splitext(file_name)
1392 1333 last_set = int(basename[-3:])
1393 1334
1394
1395 1335 self.online = online
1396 1336 self.realtime = realtime
1397 1337 self.delay = delay
@@ -1402,11 +1342,13 class JRODataReader(JRODataIO):
1402 1342 self.startTime = startTime
1403 1343 self.endTime = endTime
1404 1344
1405
1406 1345 #Added-----------------
1407 1346 self.selBlocksize = blocksize
1408 1347 self.selBlocktime = blocktime
1409 1348
1349 # Verbose-----------
1350 self.verbose = verbose
1351 self.warnings = warnings
1410 1352
1411 1353 if not(self.setNextFile()):
1412 1354 if (startDate!=None) and (endDate!=None):
@@ -1510,7 +1452,6 class JRODataReader(JRODataIO):
1510 1452 realtime=False,
1511 1453 blocksize=None,
1512 1454 blocktime=None,
1513 queue=None,
1514 1455 skip=None,
1515 1456 cursor=None,
1516 1457 warnings=True,
@@ -1518,7 +1459,6 class JRODataReader(JRODataIO):
1518 1459 verbose=True, **kwargs):
1519 1460
1520 1461 if not(self.isConfig):
1521 # self.dataOut = dataOut
1522 1462 self.setup( path=path,
1523 1463 startDate=startDate,
1524 1464 endDate=endDate,
@@ -1535,12 +1475,11 class JRODataReader(JRODataIO):
1535 1475 realtime=realtime,
1536 1476 blocksize=blocksize,
1537 1477 blocktime=blocktime,
1538 queue=queue,
1539 1478 skip=skip,
1540 1479 cursor=cursor,
1541 1480 warnings=warnings,
1542 1481 server=server,
1543 verbose=verbose, **kwargs)
1482 verbose=verbose)
1544 1483 self.isConfig = True
1545 1484 if server is None:
1546 1485 self.getData()
@@ -1792,7 +1731,7 class JRODataWriter(JRODataIO):
1792 1731
1793 1732 return 1
1794 1733
1795 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True):
1734 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1796 1735 """
1797 1736 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1798 1737
Show file before | Show file after | Hide comments
@@ -453,7 +453,7 class FitsReader(ProcessingUnit):
453 453 # self.blockIndex = 1
454 454 return 1
455 455
456 def __searchFilesOffLine(self,
456 def searchFilesOffLine(self,
457 457 path,
458 458 startDate,
459 459 endDate,
@@ -559,7 +559,7 class FitsReader(ProcessingUnit):
559 559
560 560 if not(online):
561 561 print "Searching files in offline mode ..."
562 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
562 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
563 563 startTime=startTime, endTime=endTime,
564 564 set=set, expLabel=expLabel, ext=ext,
565 565 walk=walk)
Show file before | Show file after | Hide comments
@@ -415,7 +415,7 class HFReader(ProcessingUnit):
415 415
416 416
417 417
418 def __searchFilesOffline(self,
418 def searchFilesOffLine(self,
419 419 path,
420 420 startDate,
421 421 endDate,
@@ -438,7 +438,7 class HFReader(ProcessingUnit):
438 438
439 439 return
440 440
441 def __searchFilesOnline(self,
441 def searchFilesOnLine(self,
442 442 path,
443 443 expLabel= "",
444 444 ext=None,
@@ -636,10 +636,10 class HFReader(ProcessingUnit):
636 636 if not(online):
637 637 print "Searching files in offline mode..."
638 638
639 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 640 else:
641 641 print "Searching files in online mode..."
642 self.__searchFilesOnline(path, walk,ext,set=set)
642 self.searchFilesOnLine(path, walk,ext,set=set)
643 643 if set==None:
644 644 pass
645 645 else:
@@ -647,7 +647,7 class HFReader(ProcessingUnit):
647 647
648 648 # for nTries in range(self.nTries):
649 649 #
650 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 651 #
652 652 # if fullpath:
653 653 # break
Show file before | Show file after | Hide comments
@@ -106,9 +106,9 class AMISRReader(ProcessingUnit):
106 106 #self.findFiles()
107 107 if not(online):
108 108 #Busqueda de archivos offline
109 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
109 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
110 110 else:
111 self.__searchFilesOnline(path, startDate, endDate, startTime,endTime,walk)
111 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
112 112
113 113 if not(self.filenameList):
114 114 print "There is no files into the folder: %s"%(path)
@@ -329,7 +329,7 class AMISRReader(ProcessingUnit):
329 329 self.dirnameList = new_dirnameList
330 330 return 1
331 331
332 def __searchFilesOnline(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 333 endTime=datetime.time(23,59,59),walk=True):
334 334
335 335 if endDate ==None:
@@ -349,7 +349,7 class AMISRReader(ProcessingUnit):
349 349 return
350 350
351 351
352 def __searchFilesOffline(self,
352 def searchFilesOffLine(self,
353 353 path,
354 354 startDate,
355 355 endDate,
Show file before | Show file after | Hide comments
@@ -97,7 +97,7 class ParamReader(ProcessingUnit):
97 97 self.timezone = 'lt'
98 98
99 99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 101 startTime=startTime, endTime=endTime,
102 102 ext=ext, walk=walk)
103 103
@@ -115,7 +115,7 class ParamReader(ProcessingUnit):
115 115
116 116 return
117 117
118 def __searchFilesOffLine(self,
118 def searchFilesOffLine(self,
119 119 path,
120 120 startDate=None,
121 121 endDate=None,
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now