##// END OF EJS Templates
schain-jc
RSS
Change multiSchain by MPProject
Juan C. Espinoza -
r1052:3fb7c359028f
parent child
Show More
Side by Side Unified
Context file:
r1052:3fb7c359028f Loading diff...:
Show file before | Show file after | Hide comments
@@ -9,56 +9,53 import datetime
9 9 import traceback
10 10 import math
11 11 import time
12 from multiprocessing import Process, Queue, cpu_count
13
14 import schainpy
15 import schainpy.admin
16 from schainpy.utils.log import logToFile
12 from multiprocessing import Process, cpu_count
17 13
18 14 from xml.etree.ElementTree import ElementTree, Element, SubElement, tostring
19 15 from xml.dom import minidom
20 16
17 import schainpy
18 import schainpy.admin
21 19 from schainpy.model import *
22 from time import sleep
23
24
25
26 def prettify(elem):
27 """Return a pretty-printed XML string for the Element.
28 """
29 rough_string = tostring(elem, 'utf-8')
30 reparsed = minidom.parseString(rough_string)
31 return reparsed.toprettyxml(indent=" ")
32
33 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_day=False):
34 skip = 0
35 cursor = 0
36 nFiles = None
37 processes = []
38 dt1 = datetime.datetime.strptime(startDate, '%Y/%m/%d')
39 dt2 = datetime.datetime.strptime(endDate, '%Y/%m/%d')
20 from schainpy.utils import log
21
22 DTYPES = {
23 'Voltage': '.r',
24 'Spectra': '.pdata'
25 }
26
27 def MPProject(project, n=cpu_count()):
28 '''
29 Project wrapper to run schain in n processes
30 '''
31
32 rconf = project.getReadUnitObj()
33 op = rconf.getOperationObj('run')
34 dt1 = op.getParameterValue('startDate')
35 dt2 = op.getParameterValue('endDate')
40 36 days = (dt2 - dt1).days
41
37
42 38 for day in range(days+1):
43 39 skip = 0
44 40 cursor = 0
45 q = Queue()
46 41 processes = []
47 dt = (dt1 + datetime.timedelta(day)).strftime('%Y/%m/%d')
48 firstProcess = Process(target=child, args=(cursor, skip, q, dt))
49 firstProcess.start()
50 if by_day:
51 continue
52 nFiles = q.get()
53 if nFiles==0:
42 dt = dt1 + datetime.timedelta(day)
43 dt_str = dt.strftime('%Y/%m/%d')
44 reader = JRODataReader()
45 paths, files = reader.searchFilesOffLine(path=rconf.path,
46 startDate=dt,
47 endDate=dt,
48 ext=DTYPES[rconf.datatype])
49 nFiles = len(files)
50 if nFiles == 0:
54 51 continue
55 firstProcess.terminate()
56 skip = int(math.ceil(nFiles/nProcess))
57 while True:
58 processes.append(Process(target=child, args=(cursor, skip, q, dt)))
59 processes[cursor].start()
60 if nFiles < cursor*skip:
61 break
52 skip = int(math.ceil(nFiles/n))
53 while nFiles > cursor*skip:
54 rconf.update(startDate=dt_str, endDate=dt_str, cursor=cursor,
55 skip=skip)
56 p = project.clone()
57 p.start()
58 processes.append(p)
62 59 cursor += 1
63 60
64 61 def beforeExit(exctype, value, trace):
@@ -75,7 +72,6 def multiSchain(child, nProcess=cpu_count(), startDate=None, endDate=None, by_da
75 72
76 73 time.sleep(3)
77 74
78
79 75 class ParameterConf():
80 76
81 77 id = None
@@ -112,7 +108,7 class ParameterConf():
112 108 return self.__formated_value
113 109
114 110 if value == '':
115 raise ValueError, "%s: This parameter value is empty" %self.name
111 raise ValueError, '%s: This parameter value is empty' %self.name
116 112
117 113 if format == 'list':
118 114 strList = value.split(',')
@@ -122,10 +118,10 class ParameterConf():
122 118 return self.__formated_value
123 119
124 120 if format == 'intlist':
125 """
121 '''
126 122 Example:
127 123 value = (0,1,2)
128 """
124 '''
129 125
130 126 new_value = ast.literal_eval(value)
131 127
@@ -137,10 +133,10 class ParameterConf():
137 133 return self.__formated_value
138 134
139 135 if format == 'floatlist':
140 """
136 '''
141 137 Example:
142 138 value = (0.5, 1.4, 2.7)
143 """
139 '''
144 140
145 141 new_value = ast.literal_eval(value)
146 142
@@ -170,38 +166,38 class ParameterConf():
170 166 return self.__formated_value
171 167
172 168 if format == 'pairslist':
173 """
169 '''
174 170 Example:
175 171 value = (0,1),(1,2)
176 """
172 '''
177 173
178 174 new_value = ast.literal_eval(value)
179 175
180 176 if type(new_value) not in (tuple, list):
181 raise ValueError, "%s has to be a tuple or list of pairs" %value
177 raise ValueError, '%s has to be a tuple or list of pairs' %value
182 178
183 179 if type(new_value[0]) not in (tuple, list):
184 180 if len(new_value) != 2:
185 raise ValueError, "%s has to be a tuple or list of pairs" %value
181 raise ValueError, '%s has to be a tuple or list of pairs' %value
186 182 new_value = [new_value]
187 183
188 184 for thisPair in new_value:
189 185 if len(thisPair) != 2:
190 raise ValueError, "%s has to be a tuple or list of pairs" %value
186 raise ValueError, '%s has to be a tuple or list of pairs' %value
191 187
192 188 self.__formated_value = new_value
193 189
194 190 return self.__formated_value
195 191
196 192 if format == 'multilist':
197 """
193 '''
198 194 Example:
199 195 value = (0,1,2),(3,4,5)
200 """
196 '''
201 197 multiList = ast.literal_eval(value)
202 198
203 199 if type(multiList[0]) == int:
204 multiList = ast.literal_eval("(" + value + ")")
200 multiList = ast.literal_eval('(' + value + ')')
205 201
206 202 self.__formated_value = multiList
207 203
@@ -263,9 +259,9 class ParameterConf():
263 259
264 260 def printattr(self):
265 261
266 print "Parameter[%s]: name = %s, value = %s, format = %s" %(self.id, self.name, self.value, self.format)
262 print 'Parameter[%s]: name = %s, value = %s, format = %s' %(self.id, self.name, self.value, self.format)
267 263
268 class OperationConf():
264 class OperationConf():
269 265
270 266 id = None
271 267 name = None
@@ -371,7 +367,9 class OperationConf():
371 367 self.parmConfObjList = []
372 368
373 369 def addParameter(self, name, value, format='str'):
374
370
371 if value is None:
372 return None
375 373 id = self.__getNewId()
376 374
377 375 parmConfObj = ParameterConf()
@@ -431,7 +429,7 class OperationConf():
431 429
432 430 def printattr(self):
433 431
434 print "%s[%s]: name = %s, type = %s, priority = %s" %(self.ELEMENTNAME,
432 print '%s[%s]: name = %s, type = %s, priority = %s' %(self.ELEMENTNAME,
435 433 self.id,
436 434 self.name,
437 435 self.type,
@@ -444,12 +442,11 class OperationConf():
444 442
445 443
446 444 if self.type == 'self':
447 raise ValueError, "This operation type cannot be created"
445 raise ValueError, 'This operation type cannot be created'
448 446
449 if self.type == 'plotter':
450 #Plotter(plotter_name)
447 if self.type == 'plotter':
451 448 if not plotter_queue:
452 raise ValueError, "plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)"
449 raise ValueError, 'plotter_queue is not defined. Use:\nmyProject = Project()\nmyProject.setPlotterQueue(plotter_queue)'
453 450
454 451 opObj = Plotter(self.name, plotter_queue)
455 452
@@ -564,7 +561,7 class ProcUnitConf():
564 561
565 562 #Compatible with old signal chain version
566 563 if datatype==None and name==None:
567 raise ValueError, "datatype or name should be defined"
564 raise ValueError, 'datatype or name should be defined'
568 565
569 566 if name==None:
570 567 if 'Proc' in datatype:
@@ -595,7 +592,7 class ProcUnitConf():
595 592
596 593 def addParameter(self, **kwargs):
597 594 '''
598 Add parameters to "run" operation
595 Add parameters to 'run' operation
599 596 '''
600 597 opObj = self.opConfObjList[0]
601 598
@@ -633,11 +630,11 class ProcUnitConf():
633 630 self.datatype = upElement.get('datatype')
634 631 self.inputId = upElement.get('inputId')
635 632
636 if self.ELEMENTNAME == "ReadUnit":
637 self.datatype = self.datatype.replace("Reader", "")
633 if self.ELEMENTNAME == 'ReadUnit':
634 self.datatype = self.datatype.replace('Reader', '')
638 635
639 if self.ELEMENTNAME == "ProcUnit":
640 self.datatype = self.datatype.replace("Proc", "")
636 if self.ELEMENTNAME == 'ProcUnit':
637 self.datatype = self.datatype.replace('Proc', '')
641 638
642 639 if self.inputId == 'None':
643 640 self.inputId = '0'
@@ -653,7 +650,7 class ProcUnitConf():
653 650
654 651 def printattr(self):
655 652
656 print "%s[%s]: name = %s, datatype = %s, inputId = %s" %(self.ELEMENTNAME,
653 print '%s[%s]: name = %s, datatype = %s, inputId = %s' %(self.ELEMENTNAME,
657 654 self.id,
658 655 self.name,
659 656 self.datatype,
@@ -707,17 +704,9 class ProcUnitConf():
707 704
708 705 kwargs[parmConfObj.name] = parmConfObj.getValue()
709 706
710 #ini = time.time()
711
712 #print "\tRunning the '%s' operation with %s" %(opConfObj.name, opConfObj.id)
713 707 sts = self.procUnitObj.call(opType = opConfObj.type,
714 708 opName = opConfObj.name,
715 709 opId = opConfObj.id)
716
717 # total_time = time.time() - ini
718 #
719 # if total_time > 0.002:
720 # print "%s::%s took %f seconds" %(self.name, opConfObj.name, total_time)
721 710
722 711 is_ok = is_ok or sts
723 712
@@ -762,11 +751,12 class ReadUnitConf(ProcUnitConf):
762 751
763 752 return self.ELEMENTNAME
764 753
765 def setup(self, id, name, datatype, path='', startDate="", endDate="", startTime="",
766 endTime="", parentId=None, queue=None, server=None, **kwargs):
754 def setup(self, id, name, datatype, path='', startDate='', endDate='',
755 startTime='', endTime='', parentId=None, server=None, **kwargs):
756
767 757 #Compatible with old signal chain version
768 758 if datatype==None and name==None:
769 raise ValueError, "datatype or name should be defined"
759 raise ValueError, 'datatype or name should be defined'
770 760
771 761 if name==None:
772 762 if 'Reader' in datatype:
@@ -785,39 +775,28 class ReadUnitConf(ProcUnitConf):
785 775 self.endDate = endDate
786 776 self.startTime = startTime
787 777 self.endTime = endTime
788
789 778 self.inputId = '0'
790 779 self.parentId = parentId
791 self.queue = queue
792 780 self.server = server
793 781 self.addRunOperation(**kwargs)
794 782
795 def update(self, datatype, path, startDate, endDate, startTime, endTime, parentId=None, name=None, **kwargs):
783 def update(self, **kwargs):
796 784
797 #Compatible with old signal chain version
798 if datatype==None and name==None:
799 raise ValueError, "datatype or name should be defined"
800
801 if name==None:
785 if 'datatype' in kwargs:
786 datatype = kwargs.pop('datatype')
802 787 if 'Reader' in datatype:
803 name = datatype
788 self.name = datatype
804 789 else:
805 name = '%sReader' %(datatype)
806
807 if datatype==None:
808 datatype = name.replace('Reader','')
809
810 self.datatype = datatype
811 self.name = name
812 self.path = path
813 self.startDate = startDate
814 self.endDate = endDate
815 self.startTime = startTime
816 self.endTime = endTime
790 self.name = '%sReader' %(datatype)
791 self.datatype = self.name.replace('Reader', '')
817 792
793 attrs = ('path', 'startDate', 'endDate', 'startTime', 'endTime', 'parentId')
794
795 for attr in attrs:
796 if attr in kwargs:
797 setattr(self, attr, kwargs.pop(attr))
798
818 799 self.inputId = '0'
819 self.parentId = parentId
820
821 800 self.updateRunOperation(**kwargs)
822 801
823 802 def removeOperations(self):
@@ -832,13 +811,13 class ReadUnitConf(ProcUnitConf):
832 811 opObj = self.addOperation(name = 'run', optype = 'self')
833 812
834 813 if self.server is None:
835 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
836 opObj.addParameter(name='path' , value=self.path, format='str')
837 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
838 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
839 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
840 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
841 opObj.addParameter(name='queue' , value=self.queue, format='obj')
814 opObj.addParameter(name='datatype', value=self.datatype, format='str')
815 opObj.addParameter(name='path', value=self.path, format='str')
816 opObj.addParameter(name='startDate', value=self.startDate, format='date')
817 opObj.addParameter(name='endDate', value=self.endDate, format='date')
818 opObj.addParameter(name='startTime', value=self.startTime, format='time')
819 opObj.addParameter(name='endTime', value=self.endTime, format='time')
820
842 821 for key, value in kwargs.items():
843 822 opObj.addParameter(name=key, value=value, format=type(value).__name__)
844 823 else:
@@ -849,32 +828,21 class ReadUnitConf(ProcUnitConf):
849 828
850 829 def updateRunOperation(self, **kwargs):
851 830
852 opObj = self.getOperationObj(name = 'run')
831 opObj = self.getOperationObj(name='run')
853 832 opObj.removeParameters()
854 833
855 opObj.addParameter(name='datatype' , value=self.datatype, format='str')
856 opObj.addParameter(name='path' , value=self.path, format='str')
857 opObj.addParameter(name='startDate' , value=self.startDate, format='date')
858 opObj.addParameter(name='endDate' , value=self.endDate, format='date')
859 opObj.addParameter(name='startTime' , value=self.startTime, format='time')
860 opObj.addParameter(name='endTime' , value=self.endTime, format='time')
861
834 opObj.addParameter(name='datatype', value=self.datatype, format='str')
835 opObj.addParameter(name='path', value=self.path, format='str')
836 opObj.addParameter(name='startDate', value=self.startDate, format='date')
837 opObj.addParameter(name='endDate', value=self.endDate, format='date')
838 opObj.addParameter(name='startTime', value=self.startTime, format='time')
839 opObj.addParameter(name='endTime', value=self.endTime, format='time')
840
862 841 for key, value in kwargs.items():
863 842 opObj.addParameter(name=key, value=value, format=type(value).__name__)
864 843
865 844 return opObj
866 845
867 # def makeXml(self, projectElement):
868 #
869 # procUnitElement = SubElement(projectElement, self.ELEMENTNAME)
870 # procUnitElement.set('id', str(self.id))
871 # procUnitElement.set('name', self.name)
872 # procUnitElement.set('datatype', self.datatype)
873 # procUnitElement.set('inputId', str(self.inputId))
874 #
875 # for opConfObj in self.opConfObjList:
876 # opConfObj.makeXml(procUnitElement)
877
878 846 def readXml(self, upElement):
879 847
880 848 self.id = upElement.get('id')
@@ -882,8 +850,8 class ReadUnitConf(ProcUnitConf):
882 850 self.datatype = upElement.get('datatype')
883 851 self.inputId = upElement.get('inputId')
884 852
885 if self.ELEMENTNAME == "ReadUnit":
886 self.datatype = self.datatype.replace("Reader", "")
853 if self.ELEMENTNAME == 'ReadUnit':
854 self.datatype = self.datatype.replace('Reader', '')
887 855
888 856 if self.inputId == 'None':
889 857 self.inputId = '0'
@@ -905,8 +873,9 class ReadUnitConf(ProcUnitConf):
905 873 self.endTime = opConfObj.getParameterValue('endTime')
906 874
907 875 class Project(Process):
876
908 877 id = None
909 name = None
878 # name = None
910 879 description = None
911 880 filename = None
912 881
@@ -916,17 +885,17 class Project(Process):
916 885
917 886 plotterQueue = None
918 887
919 def __init__(self, plotter_queue=None, logfile=None):
888 def __init__(self, plotter_queue=None):
889
920 890 Process.__init__(self)
921 891 self.id = None
922 self.name = None
892 # self.name = None
923 893 self.description = None
924 if logfile is not None:
925 logToFile(logfile)
894
926 895 self.plotterQueue = plotter_queue
927 896
928 897 self.procUnitConfObjDict = {}
929
898
930 899 def __getNewId(self):
931 900
932 901 idList = self.procUnitConfObjDict.keys()
@@ -972,18 +941,28 class Project(Process):
972 941
973 942 self.procUnitConfObjDict = newProcUnitConfObjDict
974 943
975 def setup(self, id, name, description):
944 def setup(self, id, name='', description=''):
976 945
946 print
947 print '*'*60
948 print ' Starting SIGNAL CHAIN PROCESSING v%s ' % schainpy.__version__
949 print '*'*60
950 print
977 951 self.id = str(id)
978 self.name = name
979 952 self.description = description
980 953
981 954 def update(self, name, description):
982 955
983 self.name = name
984 956 self.description = description
985 957
958 def clone(self):
959
960 p = Project()
961 p.procUnitConfObjDict = self.procUnitConfObjDict
962 return p
963
986 964 def addReadUnit(self, id=None, datatype=None, name=None, **kwargs):
965
987 966 if id is None:
988 967 idReadUnit = self.__getNewId()
989 968 else:
@@ -1021,7 +1000,7 class Project(Process):
1021 1000 def getReadUnitObj(self):
1022 1001
1023 1002 for obj in self.procUnitConfObjDict.values():
1024 if obj.getElementName() == "ReadUnit":
1003 if obj.getElementName() == 'ReadUnit':
1025 1004 return obj
1026 1005
1027 1006 return None
@@ -1066,20 +1045,20 class Project(Process):
1066 1045 if self.filename:
1067 1046 filename = self.filename
1068 1047 else:
1069 filename = "schain.xml"
1048 filename = 'schain.xml'
1070 1049
1071 1050 if not filename:
1072 print "filename has not been defined. Use setFilename(filename) for do it."
1051 print 'filename has not been defined. Use setFilename(filename) for do it.'
1073 1052 return 0
1074 1053
1075 1054 abs_file = os.path.abspath(filename)
1076 1055
1077 1056 if not os.access(os.path.dirname(abs_file), os.W_OK):
1078 print "No write permission on %s" %os.path.dirname(abs_file)
1057 print 'No write permission on %s' %os.path.dirname(abs_file)
1079 1058 return 0
1080 1059
1081 1060 if os.path.isfile(abs_file) and not(os.access(abs_file, os.W_OK)):
1082 print "File %s already exists and it could not be overwriten" %abs_file
1061 print 'File %s already exists and it could not be overwriten' %abs_file
1083 1062 return 0
1084 1063
1085 1064 self.makeXml()
@@ -1093,13 +1072,13 class Project(Process):
1093 1072 def readXml(self, filename = None):
1094 1073
1095 1074 if not filename:
1096 print "filename is not defined"
1075 print 'filename is not defined'
1097 1076 return 0
1098 1077
1099 1078 abs_file = os.path.abspath(filename)
1100 1079
1101 1080 if not os.path.isfile(abs_file):
1102 print "%s file does not exist" %abs_file
1081 print '%s file does not exist' %abs_file
1103 1082 return 0
1104 1083
1105 1084 self.projectElement = None
@@ -1108,7 +1087,7 class Project(Process):
1108 1087 try:
1109 1088 self.projectElement = ElementTree().parse(abs_file)
1110 1089 except:
1111 print "Error reading %s, verify file format" %filename
1090 print 'Error reading %s, verify file format' %filename
1112 1091 return 0
1113 1092
1114 1093 self.project = self.projectElement.tag
@@ -1145,10 +1124,10 class Project(Process):
1145 1124
1146 1125 def printattr(self):
1147 1126
1148 print "Project[%s]: name = %s, description = %s" %(self.id,
1149 self.name,
1150 self.description)
1151
1127 print 'Project[%s]: name = %s, description = %s' %(self.id,
1128 self.name,
1129 self.description)
1130
1152 1131 for procUnitConfObj in self.procUnitConfObjDict.values():
1153 1132 procUnitConfObj.printattr()
1154 1133
@@ -1179,7 +1158,7 class Project(Process):
1179 1158
1180 1159 self.__connect(puObjIN, thisPUObj)
1181 1160
1182 def __handleError(self, procUnitConfObj, send_email=True):
1161 def __handleError(self, procUnitConfObj, send_email=False):
1183 1162
1184 1163 import socket
1185 1164
@@ -1187,33 +1166,33 class Project(Process):
1187 1166 sys.exc_info()[1],
1188 1167 sys.exc_info()[2])
1189 1168
1190 print "***** Error occurred in %s *****" %(procUnitConfObj.name)
1191 print "***** %s" %err[-1]
1169 print '***** Error occurred in %s *****' %(procUnitConfObj.name)
1170 print '***** %s' %err[-1]
1192 1171
1193 message = "".join(err)
1172 message = ''.join(err)
1194 1173
1195 1174 sys.stderr.write(message)
1196 1175
1197 1176 if not send_email:
1198 1177 return
1199 1178
1200 subject = "SChain v%s: Error running %s\n" %(schainpy.__version__, procUnitConfObj.name)
1179 subject = 'SChain v%s: Error running %s\n' %(schainpy.__version__, procUnitConfObj.name)
1201 1180
1202 subtitle = "%s: %s\n" %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1203 subtitle += "Hostname: %s\n" %socket.gethostbyname(socket.gethostname())
1204 subtitle += "Working directory: %s\n" %os.path.abspath("./")
1205 subtitle += "Configuration file: %s\n" %self.filename
1206 subtitle += "Time: %s\n" %str(datetime.datetime.now())
1181 subtitle = '%s: %s\n' %(procUnitConfObj.getElementName() ,procUnitConfObj.name)
1182 subtitle += 'Hostname: %s\n' %socket.gethostbyname(socket.gethostname())
1183 subtitle += 'Working directory: %s\n' %os.path.abspath('./')
1184 subtitle += 'Configuration file: %s\n' %self.filename
1185 subtitle += 'Time: %s\n' %str(datetime.datetime.now())
1207 1186
1208 1187 readUnitConfObj = self.getReadUnitObj()
1209 1188 if readUnitConfObj:
1210 subtitle += "\nInput parameters:\n"
1211 subtitle += "[Data path = %s]\n" %readUnitConfObj.path
1212 subtitle += "[Data type = %s]\n" %readUnitConfObj.datatype
1213 subtitle += "[Start date = %s]\n" %readUnitConfObj.startDate
1214 subtitle += "[End date = %s]\n" %readUnitConfObj.endDate
1215 subtitle += "[Start time = %s]\n" %readUnitConfObj.startTime
1216 subtitle += "[End time = %s]\n" %readUnitConfObj.endTime
1189 subtitle += '\nInput parameters:\n'
1190 subtitle += '[Data path = %s]\n' %readUnitConfObj.path
1191 subtitle += '[Data type = %s]\n' %readUnitConfObj.datatype
1192 subtitle += '[Start date = %s]\n' %readUnitConfObj.startDate
1193 subtitle += '[End date = %s]\n' %readUnitConfObj.endDate
1194 subtitle += '[Start time = %s]\n' %readUnitConfObj.startTime
1195 subtitle += '[End time = %s]\n' %readUnitConfObj.endTime
1217 1196
1218 1197 adminObj = schainpy.admin.SchainNotify()
1219 1198 adminObj.sendAlert(message=message,
@@ -1228,15 +1207,15 class Project(Process):
1228 1207 return 0
1229 1208
1230 1209 def runController(self):
1231 """
1210 '''
1232 1211 returns 0 when this process has been stopped, 1 otherwise
1233 """
1212 '''
1234 1213
1235 1214 if self.isPaused():
1236 print "Process suspended"
1215 print 'Process suspended'
1237 1216
1238 1217 while True:
1239 sleep(0.1)
1218 time.sleep(0.1)
1240 1219
1241 1220 if not self.isPaused():
1242 1221 break
@@ -1244,10 +1223,10 class Project(Process):
1244 1223 if self.isStopped():
1245 1224 break
1246 1225
1247 print "Process reinitialized"
1226 print 'Process reinitialized'
1248 1227
1249 1228 if self.isStopped():
1250 print "Process stopped"
1229 print 'Process stopped'
1251 1230 return 0
1252 1231
1253 1232 return 1
@@ -1258,29 +1237,23 class Project(Process):
1258 1237
1259 1238 def setPlotterQueue(self, plotter_queue):
1260 1239
1261 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1240 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1262 1241
1263 1242 def getPlotterQueue(self):
1264 1243
1265 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1244 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1266 1245
1267 1246 def useExternalPlotter(self):
1268 1247
1269 raise NotImplementedError, "Use schainpy.controller_api.ControllerThread instead Project class"
1248 raise NotImplementedError, 'Use schainpy.controller_api.ControllerThread instead Project class'
1270 1249
1250 def run(self):
1271 1251
1272 def run(self, filename=None):
1252 log.success('Starting {}'.format(self.name))
1273 1253
1274 # self.writeXml(filename)
1275 1254 self.createObjects()
1276 1255 self.connectObjects()
1277 1256
1278 print
1279 print "*"*60
1280 print " Starting SIGNAL CHAIN PROCESSING v%s " %schainpy.__version__
1281 print "*"*60
1282 print
1283
1284 1257 keyList = self.procUnitConfObjDict.keys()
1285 1258 keyList.sort()
1286 1259
@@ -1289,7 +1262,6 class Project(Process):
1289 1262 is_ok = False
1290 1263
1291 1264 for procKey in keyList:
1292 # print "Running the '%s' process with %s" %(procUnitConfObj.name, procUnitConfObj.id)
1293 1265
1294 1266 procUnitConfObj = self.procUnitConfObjDict[procKey]
1295 1267
@@ -1300,19 +1272,18 class Project(Process):
1300 1272 is_ok = False
1301 1273 break
1302 1274 except ValueError, e:
1303 sleep(0.5)
1275 time.sleep(0.5)
1304 1276 self.__handleError(procUnitConfObj, send_email=True)
1305 1277 is_ok = False
1306 1278 break
1307 1279 except:
1308 sleep(0.5)
1280 time.sleep(0.5)
1309 1281 self.__handleError(procUnitConfObj)
1310 1282 is_ok = False
1311 1283 break
1312 1284
1313 1285 #If every process unit finished so end process
1314 1286 if not(is_ok):
1315 # print "Every process unit have finished"
1316 1287 break
1317 1288
1318 1289 if not self.runController():
@@ -1322,3 +1293,5 class Project(Process):
1322 1293 for procKey in keyList:
1323 1294 procUnitConfObj = self.procUnitConfObjDict[procKey]
1324 1295 procUnitConfObj.close()
1296
1297 log.success('{} finished'.format(self.name))
Show file before | Show file after | Hide comments
@@ -6218,3 +6218,6 class ShowMeConsole(QtCore.QObject):
6218 6218 text = text[:-1]
6219 6219
6220 6220 self.textWritten.emit(str(text))
6221
6222 def flush(self):
6223 pass
Show file before | Show file after | Hide comments
@@ -267,7 +267,7 class AMISRReader(ProcessingUnit):
267 267 self.dirnameList = new_dirnameList
268 268 return 1
269 269
270 def __searchFilesOnline(self,
270 def searchFilesOnLine(self,
271 271 path,
272 272 walk=True):
273 273
@@ -287,7 +287,7 class AMISRReader(ProcessingUnit):
287 287 return
288 288
289 289
290 def __searchFilesOffline(self,
290 def searchFilesOffLine(self,
291 291 path,
292 292 startDate,
293 293 endDate,
@@ -494,9 +494,9 class AMISRReader(ProcessingUnit):
494 494 self.online = online
495 495 if not(online):
496 496 #Busqueda de archivos offline
497 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
497 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
498 498 else:
499 self.__searchFilesOnline(path, walk)
499 self.searchFilesOnLine(path, walk)
500 500
501 501 if not(self.filenameList):
502 502 print "There is no files into the folder: %s"%(path)
Show file before | Show file after | Hide comments
This diff has been collapsed as it changes many lines, (543 lines changed) Show them Hide them
@@ -541,8 +541,7 class JRODataIO:
541 541 return inspect.getargspec(self.run).args
542 542
543 543 class JRODataReader(JRODataIO):
544
545 firstTime = True
544
546 545 online = 0
547 546
548 547 realtime = 0
@@ -578,8 +577,7 class JRODataReader(JRODataIO):
578 577 selBlocksize = None
579 578
580 579 selBlocktime = None
581
582 onlineWithDate = False
580
583 581 def __init__(self):
584 582
585 583 """
@@ -603,19 +601,19 class JRODataReader(JRODataIO):
603 601
604 602 raise NotImplementedError
605 603
606 def __searchFilesOffLine(self,
607 path,
608 startDate=None,
609 endDate=None,
610 startTime=datetime.time(0,0,0),
611 endTime=datetime.time(23,59,59),
612 set=None,
613 expLabel='',
614 ext='.r',
615 queue=None,
616 cursor=None,
617 skip=None,
618 walk=True):
604 def searchFilesOffLine(self,
605 path,
606 startDate=None,
607 endDate=None,
608 startTime=datetime.time(0,0,0),
609 endTime=datetime.time(23,59,59),
610 set=None,
611 expLabel='',
612 ext='.r',
613 cursor=None,
614 skip=None,
615 walk=True):
616
619 617 self.filenameList = []
620 618 self.datetimeList = []
621 619
@@ -624,8 +622,7 class JRODataReader(JRODataIO):
624 622 dateList, pathList = self.findDatafiles(path, startDate, endDate, expLabel, ext, walk, include_path=True)
625 623
626 624 if dateList == []:
627 # print "[Reading] Date range selected invalid [%s - %s]: No *%s files in %s)" %(startDate, endDate, ext, path)
628 return None, None
625 return [], []
629 626
630 627 if len(dateList) > 1:
631 628 print "[Reading] Data found for date range [%s - %s]: total days = %d" %(startDate, endDate, len(dateList))
@@ -636,18 +633,15 class JRODataReader(JRODataIO):
636 633 datetimeList = []
637 634
638 635 for thisPath in pathList:
639 # thisPath = pathList[pathDict[file]]
640
636
641 637 fileList = glob.glob1(thisPath, "*%s" %ext)
642 638 fileList.sort()
643 639
644 640 skippedFileList = []
645 641
646 642 if cursor is not None and skip is not None:
647 # if cursor*skip > len(fileList):
643
648 644 if skip == 0:
649 if queue is not None:
650 queue.put(len(fileList))
651 645 skippedFileList = []
652 646 else:
653 647 skippedFileList = fileList[cursor*skip: cursor*skip + skip]
@@ -672,44 +666,43 class JRODataReader(JRODataIO):
672 666
673 667 if not(filenameList):
674 668 print "[Reading] Time range selected invalid [%s - %s]: No *%s files in %s)" %(startTime, endTime, ext, path)
675 return None, None
669 return [], []
676 670
677 671 print "[Reading] %d file(s) was(were) found in time range: %s - %s" %(len(filenameList), startTime, endTime)
678 672 print
679 673
680 for i in range(len(filenameList)):
681 print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
674 # for i in range(len(filenameList)):
675 # print "[Reading] %s -> [%s]" %(filenameList[i], datetimeList[i].ctime())
682 676
683 677 self.filenameList = filenameList
684 self.datetimeList = datetimeList
678 self.datetimeList = datetimeList
679
685 680 return pathList, filenameList
686 681
687 def __searchFilesOnLine(self, path, expLabel="", ext=None, walk=True, set=None, startDate=None, startTime=None):
688
682 def __searchFilesOnLine(self, path, expLabel = "", ext = None, walk=True, set=None):
683
689 684 """
690 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
691 devuelve el archivo encontrado ademas de otros datos.
692
693 Input:
694 path : carpeta donde estan contenidos los files que contiene data
695
696 expLabel : Nombre del subexperimento (subfolder)
697
698 ext : extension de los files
699
700 walk : Si es habilitado no realiza busquedas dentro de los subdirectorios (doypath)
701
702 Return:
703 directory : eL directorio donde esta el file encontrado
704 filename : el ultimo file de una determinada carpeta
705 year : el anho
706 doy : el numero de dia del anho
707 set : el set del archivo
708
709
685 Busca el ultimo archivo de la ultima carpeta (determinada o no por startDateTime) y
686 devuelve el archivo encontrado ademas de otros datos.
687
688 Input:
689 path : carpeta donde estan contenidos los files que contiene data
690
691 expLabel : Nombre del subexperimento (subfolder)
692
693 ext : extension de los files
694
695 walk : Si es habilitado no realiza busquedas dentro de los ubdirectorios (doypath)
696
697 Return:
698 directory : eL directorio donde esta el file encontrado
699 filename : el ultimo file de una determinada carpeta
700 year : el anho
701 doy : el numero de dia del anho
702 set : el set del archivo
703
704
710 705 """
711 pathList = None
712 filenameList = None
713 706 if not os.path.isdir(path):
714 707 return None, None, None, None, None, None
715 708
@@ -719,7 +712,7 class JRODataReader(JRODataIO):
719 712 fullpath = path
720 713 foldercounter = 0
721 714 else:
722 # Filtra solo los directorios
715 #Filtra solo los directorios
723 716 for thisPath in os.listdir(path):
724 717 if not os.path.isdir(os.path.join(path,thisPath)):
725 718 continue
@@ -755,7 +748,7 class JRODataReader(JRODataIO):
755 748
756 749 year = int( filename[1:5] )
757 750 doy = int( filename[5:8] )
758 set = int( filename[8:11] )
751 set = int( filename[8:11] )
759 752
760 753 return fullpath, foldercounter, filename, year, doy, set
761 754
@@ -767,7 +760,7 class JRODataReader(JRODataIO):
767 760 idFile += 1
768 761 if not(idFile < len(self.filenameList)):
769 762 self.flagNoMoreFiles = 1
770 # print "[Reading] No more Files"
763 # print "[Reading] No more Files"
771 764 return 0
772 765
773 766 filename = self.filenameList[idFile]
@@ -785,32 +778,31 class JRODataReader(JRODataIO):
785 778 self.fileSize = fileSize
786 779 self.fp = fp
787 780
788 #print "[Reading] Setting the file: %s"%self.filename
781 # print "[Reading] Setting the file: %s"%self.filename
789 782
790 783 return 1
791 784
792 785 def __setNextFileOnline(self):
793 786 """
794 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
795 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
796 siguientes.
797
798 Affected:
799 self.flagIsNewFile
800 self.filename
801 self.fileSize
802 self.fp
803 self.set
804 self.flagNoMoreFiles
805
806 Return:
807 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
808 1 : si el file fue abierto con exito y esta listo a ser leido
809
810 Excepciones:
811 Si un determinado file no puede ser abierto
812 """
813
787 Busca el siguiente file que tenga suficiente data para ser leida, dentro de un folder especifico, si
788 no encuentra un file valido espera un tiempo determinado y luego busca en los posibles n files
789 siguientes.
790
791 Affected:
792 self.flagIsNewFile
793 self.filename
794 self.fileSize
795 self.fp
796 self.set
797 self.flagNoMoreFiles
798
799 Return:
800 0 : si luego de una busqueda del siguiente file valido este no pudo ser encontrado
801 1 : si el file fue abierto con exito y esta listo a ser leido
802
803 Excepciones:
804 Si un determinado file no puede ser abierto
805 """
814 806 nFiles = 0
815 807 fileOk_flag = False
816 808 firstTime_flag = True
@@ -869,48 +861,27 class JRODataReader(JRODataIO):
869 861 if self.fp != None: self.fp.close()
870 862 self.fp = open(fullfilename, 'rb')
871 863 self.flagNoMoreFiles = 0
872 # print '[Reading] Setting the file: %s' % fullfilename
864 # print '[Reading] Setting the file: %s' % fullfilename
873 865 else:
874 866 self.fileSize = 0
875 867 self.filename = None
876 868 self.flagIsNewFile = 0
877 869 self.fp = None
878 870 self.flagNoMoreFiles = 1
879 # print '[Reading] No more files to read'
871 # print '[Reading] No more files to read'
880 872
881 873 return fileOk_flag
882 874
883 875 def setNextFile(self):
884 876 if self.fp != None:
885 877 self.fp.close()
878
886 879 if self.online:
887 880 newFile = self.__setNextFileOnline()
888 881 else:
889 882 newFile = self.__setNextFileOffline()
883
890 884 if not(newFile):
891 if self.onlineWithDate is True:
892 self.onlineWithDate=False
893 self.online = True
894 self.firstTime = False
895 self.setup(
896 path=self.path,
897 startDate=self.startDate,
898 endDate=self.endDate,
899 startTime=self.startTime ,
900 endTime=self.endTime,
901 set=self.set,
902 expLabel=self.expLabel,
903 ext=self.ext,
904 online=self.online,
905 delay=self.delay,
906 walk=self.walk,
907 getblock=self.getblock,
908 nTxs=self.nTxs,
909 realtime=self.realtime,
910 blocksize=self.blocksize,
911 blocktime=self.blocktime
912 )
913 return 1
914 885 print '[Reading] No more files to read'
915 886 return 0
916 887
@@ -951,7 +922,7 class JRODataReader(JRODataIO):
951 922 return 1
952 923
953 924 if self.fileSize == self.fileSizeByHeader:
954 # self.flagEoF = True
925 # self.flagEoF = True
955 926 return 0
956 927
957 928 print "[Reading] Waiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1)
@@ -1005,13 +976,13 class JRODataReader(JRODataIO):
1005 976 else:
1006 977 self.fp.seek(self.fp.tell() - neededsize)
1007 978 break
1008
1009 # csize = self.fileSize - self.fp.tell()
1010 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
1011 # factor = int(csize/neededsize)
1012 # if factor > 0:
1013 # self.fp.seek(self.fp.tell() + factor*neededsize)
1014
979
980 # csize = self.fileSize - self.fp.tell()
981 # neededsize = self.processingHeaderObj.blockSize + self.basicHeaderSize
982 # factor = int(csize/neededsize)
983 # if factor > 0:
984 # self.fp.seek(self.fp.tell() + factor*neededsize)
985
1015 986 self.flagIsNewFile = 0
1016 987 self.__isFirstTimeOnline = 0
1017 988
@@ -1019,10 +990,10 class JRODataReader(JRODataIO):
1019 990 #if self.server is None:
1020 991 if self.fp == None:
1021 992 return 0
1022
1023 # if self.online:
1024 # self.__jumpToLastBlock()
1025
993
994 # if self.online:
995 # self.__jumpToLastBlock()
996
1026 997 if self.flagIsNewFile:
1027 998 self.lastUTTime = self.basicHeaderObj.utc
1028 999 return 1
@@ -1065,12 +1036,14 class JRODataReader(JRODataIO):
1065 1036
1066 1037 #Skip block out of startTime and endTime
1067 1038 while True:
1068 if not(self.__setNewBlock()):
1069 print 'returning'
1039 if not(self.__setNewBlock()):
1070 1040 return 0
1041
1071 1042 if not(self.readBlock()):
1072 1043 return 0
1044
1073 1045 self.getBasicHeader()
1046
1074 1047 if not isTimeInRange(self.dataOut.datatime.time(), self.startTime, self.endTime):
1075 1048
1076 1049 print "[Reading] Block No. %d/%d -> %s [Skipping]" %(self.nReadBlocks,
@@ -1082,8 +1055,8 class JRODataReader(JRODataIO):
1082 1055
1083 1056 if self.verbose:
1084 1057 print "[Reading] Block No. %d/%d -> %s" %(self.nReadBlocks,
1085 self.processingHeaderObj.dataBlocksPerFile,
1086 self.dataOut.datatime.ctime())
1058 self.processingHeaderObj.dataBlocksPerFile,
1059 self.dataOut.datatime.ctime())
1087 1060 return 1
1088 1061
1089 1062 def __readFirstHeader(self):
@@ -1114,8 +1087,8 class JRODataReader(JRODataIO):
1114 1087 self.dtype = datatype_str
1115 1088 #self.ippSeconds = 2 * 1000 * self.radarControllerHeaderObj.ipp / self.c
1116 1089 self.fileSizeByHeader = self.processingHeaderObj.dataBlocksPerFile * self.processingHeaderObj.blockSize + self.firstHeaderSize + self.basicHeaderSize*(self.processingHeaderObj.dataBlocksPerFile - 1)
1117 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1118 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1090 # self.dataOut.channelList = numpy.arange(self.systemHeaderObj.numChannels)
1091 # self.dataOut.channelIndexList = numpy.arange(self.systemHeaderObj.numChannels)
1119 1092 self.getBlockDimension()
1120 1093
1121 1094 def __verifyFile(self, filename, msgFlag=True):
@@ -1277,13 +1250,13 class JRODataReader(JRODataIO):
1277 1250
1278 1251 def setup(self,
1279 1252 path=None,
1280 startDate=None,
1281 endDate=None,
1282 startTime=datetime.time(0,0,0),
1283 endTime=datetime.time(23,59,59),
1284 set=None,
1285 expLabel = "",
1286 ext = None,
1253 startDate=None,
1254 endDate=None,
1255 startTime=datetime.time(0,0,0),
1256 endTime=datetime.time(23,59,59),
1257 set=None,
1258 expLabel = "",
1259 ext = None,
1287 1260 online = False,
1288 1261 delay = 60,
1289 1262 walk = True,
@@ -1292,139 +1265,108 class JRODataReader(JRODataIO):
1292 1265 realtime=False,
1293 1266 blocksize=None,
1294 1267 blocktime=None,
1268 skip=None,
1269 cursor=None,
1270 warnings=True,
1295 1271 verbose=True,
1296 **kwargs):
1272 server=None):
1273 if server is not None:
1274 if 'tcp://' in server:
1275 address = server
1276 else:
1277 address = 'ipc:///tmp/%s' % server
1278 self.server = address
1279 self.context = zmq.Context()
1280 self.receiver = self.context.socket(zmq.PULL)
1281 self.receiver.connect(self.server)
1282 time.sleep(0.5)
1283 print '[Starting] ReceiverData from {}'.format(self.server)
1284 else:
1285 self.server = None
1286 if path == None:
1287 raise ValueError, "[Reading] The path is not valid"
1297 1288
1298 if path == None:
1299 raise ValueError, "[Reading] The path is not valid"
1300
1289 if ext == None:
1290 ext = self.ext
1301 1291
1302 if ext == None:
1303 ext = self.ext
1304
1305 self.verbose=verbose
1306 self.path = path
1307 self.startDate = startDate
1308 self.endDate = endDate
1309 self.startTime = startTime
1310 self.endTime = endTime
1311 self.set = set
1312 self.expLabel = expLabel
1313 self.ext = ext
1314 self.online = online
1315 self.delay = delay
1316 self.walk = walk
1317 self.getblock = getblock
1318 self.nTxs = nTxs
1319 self.realtime = realtime
1320 self.blocksize = blocksize
1321 self.blocktime = blocktime
1322
1323
1324 if self.firstTime is True:
1325 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1292 if online:
1293 print "[Reading] Searching files in online mode..."
1294
1295 for nTries in range( self.nTries ):
1296 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path, expLabel=expLabel, ext=ext, walk=walk, set=set)
1297
1298 if fullpath:
1299 break
1300
1301 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1302 sleep( self.delay )
1303
1304 if not(fullpath):
1305 print "[Reading] There 'isn't any valid file in %s" % path
1306 return
1307
1308 self.year = year
1309 self.doy = doy
1310 self.set = set - 1
1311 self.path = path
1312 self.foldercounter = foldercounter
1313 last_set = None
1314 else:
1315 print "[Reading] Searching files in offline mode ..."
1316 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1326 1317 startTime=startTime, endTime=endTime,
1327 1318 set=set, expLabel=expLabel, ext=ext,
1328 walk=walk)
1329 if filenameList is not None: filenameList = filenameList[:-1]
1319 walk=walk, cursor=cursor,
1320 skip=skip)
1330 1321
1331 if pathList is not None and filenameList is not None and online:
1332 self.onlineWithDate = True
1333 online = False
1322 if not(pathList):
1334 1323 self.fileIndex = -1
1335 self.pathList = pathList
1336 self.filenameList = filenameList
1337 file_name = os.path.basename(filenameList[-1])
1338 basename, ext = os.path.splitext(file_name)
1339 last_set = int(basename[-3:])
1340
1341 if online:
1342 print "[Reading] Searching files in online mode..."
1343
1344 for nTries in range(self.nTries):
1345 fullpath, foldercounter, file, year, doy, set = self.__searchFilesOnLine(path=path,
1346 expLabel=expLabel,
1347 ext=ext,
1348 walk=walk,
1349 startDate=startDate,
1350 startTime=startTime,
1351 set=set)
1352
1353 if fullpath:
1354 break
1355 print '[Reading] Waiting %0.2f sec for an valid file in %s: try %02d ...' % (self.delay, path, nTries+1)
1356 sleep( self.delay )
1357
1358 if not(fullpath):
1359 print "[Reading] There 'isn't any valid file in %s" % path
1360 return
1361
1362 self.year = year
1363 self.doy = doy
1364 self.set = set - 1
1365 self.path = path
1366 self.foldercounter = foldercounter
1367 last_set = None
1368 else:
1369 print "[Reading] Searching files in offline mode ..."
1370 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
1371 startTime=startTime, endTime=endTime,
1372 set=set, expLabel=expLabel, ext=ext,
1373 walk=walk)
1374
1375 if not(pathList):
1376 # print "[Reading] No *%s files in %s (%s - %s)"%(ext, path,
1377 # datetime.datetime.combine(startDate,startTime).ctime(),
1378 # datetime.datetime.combine(endDate,endTime).ctime())
1379
1380 # sys.exit(-1)
1381
1382 self.fileIndex = -1
1383 self.pathList = []
1384 self.filenameList = []
1385 return
1386
1387 self.fileIndex = -1
1388 self.pathList = pathList
1389 self.filenameList = filenameList
1390 file_name = os.path.basename(filenameList[-1])
1391 basename, ext = os.path.splitext(file_name)
1392 last_set = int(basename[-3:])
1393
1394
1395 self.online = online
1396 self.realtime = realtime
1397 self.delay = delay
1398 ext = ext.lower()
1399 self.ext = ext
1400 self.getByBlock = getblock
1401 self.nTxs = nTxs
1402 self.startTime = startTime
1403 self.endTime = endTime
1404
1405
1406 #Added-----------------
1407 self.selBlocksize = blocksize
1408 self.selBlocktime = blocktime
1409
1410
1411 if not(self.setNextFile()):
1412 if (startDate!=None) and (endDate!=None):
1413 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1414 elif startDate != None:
1415 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1416 else:
1417 print "[Reading] No files"
1418
1324 self.pathList = []
1325 self.filenameList = []
1326 return
1327
1419 1328 self.fileIndex = -1
1420 self.pathList = []
1421 self.filenameList = []
1422 return
1329 self.pathList = pathList
1330 self.filenameList = filenameList
1331 file_name = os.path.basename(filenameList[-1])
1332 basename, ext = os.path.splitext(file_name)
1333 last_set = int(basename[-3:])
1334
1335 self.online = online
1336 self.realtime = realtime
1337 self.delay = delay
1338 ext = ext.lower()
1339 self.ext = ext
1340 self.getByBlock = getblock
1341 self.nTxs = nTxs
1342 self.startTime = startTime
1343 self.endTime = endTime
1344
1345 #Added-----------------
1346 self.selBlocksize = blocksize
1347 self.selBlocktime = blocktime
1348
1349 # Verbose-----------
1350 self.verbose = verbose
1351 self.warnings = warnings
1352
1353 if not(self.setNextFile()):
1354 if (startDate!=None) and (endDate!=None):
1355 print "[Reading] No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime())
1356 elif startDate != None:
1357 print "[Reading] No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime())
1358 else:
1359 print "[Reading] No files"
1360
1361 self.fileIndex = -1
1362 self.pathList = []
1363 self.filenameList = []
1364 return
1423 1365
1424 # self.getBasicHeader()
1366 # self.getBasicHeader()
1425 1367
1426 if last_set != None:
1427 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1368 if last_set != None:
1369 self.dataOut.last_block = last_set * self.processingHeaderObj.dataBlocksPerFile + self.basicHeaderObj.dataBlock
1428 1370 return
1429 1371
1430 1372 def getBasicHeader(self):
@@ -1442,10 +1384,10 class JRODataReader(JRODataIO):
1442 1384 self.dataOut.useLocalTime = self.basicHeaderObj.useLocalTime
1443 1385
1444 1386 self.dataOut.ippSeconds = self.radarControllerHeaderObj.ippSeconds/self.nTxs
1445
1446 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1447
1448
1387
1388 # self.dataOut.nProfiles = self.processingHeaderObj.profilesPerBlock*self.nTxs
1389
1390
1449 1391 def getFirstHeader(self):
1450 1392
1451 1393 raise NotImplementedError
@@ -1494,53 +1436,50 class JRODataReader(JRODataIO):
1494 1436 self.__printInfo = False
1495 1437
1496 1438 def run(self,
1497 path=None,
1498 startDate=None,
1499 endDate=None,
1500 startTime=datetime.time(0,0,0),
1501 endTime=datetime.time(23,59,59),
1502 set=None,
1503 expLabel = "",
1504 ext = None,
1505 online = False,
1506 delay = 60,
1507 walk = True,
1508 getblock = False,
1509 nTxs = 1,
1510 realtime=False,
1511 blocksize=None,
1512 blocktime=None,
1513 queue=None,
1514 skip=None,
1515 cursor=None,
1516 warnings=True,
1517 server=None,
1518 verbose=True, **kwargs):
1439 path=None,
1440 startDate=None,
1441 endDate=None,
1442 startTime=datetime.time(0,0,0),
1443 endTime=datetime.time(23,59,59),
1444 set=None,
1445 expLabel = "",
1446 ext = None,
1447 online = False,
1448 delay = 60,
1449 walk = True,
1450 getblock = False,
1451 nTxs = 1,
1452 realtime=False,
1453 blocksize=None,
1454 blocktime=None,
1455 skip=None,
1456 cursor=None,
1457 warnings=True,
1458 server=None,
1459 verbose=True, **kwargs):
1519 1460
1520 1461 if not(self.isConfig):
1521 # self.dataOut = dataOut
1522 self.setup( path=path,
1523 startDate=startDate,
1524 endDate=endDate,
1525 startTime=startTime,
1526 endTime=endTime,
1527 set=set,
1528 expLabel=expLabel,
1529 ext=ext,
1530 online=online,
1531 delay=delay,
1532 walk=walk,
1533 getblock=getblock,
1534 nTxs=nTxs,
1535 realtime=realtime,
1536 blocksize=blocksize,
1537 blocktime=blocktime,
1538 queue=queue,
1539 skip=skip,
1540 cursor=cursor,
1541 warnings=warnings,
1542 server=server,
1543 verbose=verbose, **kwargs)
1462 self.setup(path=path,
1463 startDate=startDate,
1464 endDate=endDate,
1465 startTime=startTime,
1466 endTime=endTime,
1467 set=set,
1468 expLabel=expLabel,
1469 ext=ext,
1470 online=online,
1471 delay=delay,
1472 walk=walk,
1473 getblock=getblock,
1474 nTxs=nTxs,
1475 realtime=realtime,
1476 blocksize=blocksize,
1477 blocktime=blocktime,
1478 skip=skip,
1479 cursor=cursor,
1480 warnings=warnings,
1481 server=server,
1482 verbose=verbose)
1544 1483 self.isConfig = True
1545 1484 if server is None:
1546 1485 self.getData()
@@ -1663,9 +1602,9 class JRODataWriter(JRODataIO):
1663 1602 Return:
1664 1603 None
1665 1604 """
1666
1667 # CALCULAR PARAMETROS
1668
1605
1606 # CALCULAR PARAMETROS
1607
1669 1608 sizeLongHeader = self.systemHeaderObj.size + self.radarControllerHeaderObj.size + self.processingHeaderObj.size
1670 1609 self.basicHeaderObj.size = self.basicHeaderSize + sizeLongHeader
1671 1610
@@ -1792,7 +1731,7 class JRODataWriter(JRODataIO):
1792 1731
1793 1732 return 1
1794 1733
1795 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4, verbose=True):
1734 def setup(self, dataOut, path, blocksPerFile, profilesPerBlock=64, set=None, ext=None, datatype=4):
1796 1735 """
1797 1736 Setea el tipo de formato en la cual sera guardada la data y escribe el First Header
1798 1737
Show file before | Show file after | Hide comments
@@ -453,7 +453,7 class FitsReader(ProcessingUnit):
453 453 # self.blockIndex = 1
454 454 return 1
455 455
456 def __searchFilesOffLine(self,
456 def searchFilesOffLine(self,
457 457 path,
458 458 startDate,
459 459 endDate,
@@ -559,7 +559,7 class FitsReader(ProcessingUnit):
559 559
560 560 if not(online):
561 561 print "Searching files in offline mode ..."
562 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
562 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
563 563 startTime=startTime, endTime=endTime,
564 564 set=set, expLabel=expLabel, ext=ext,
565 565 walk=walk)
Show file before | Show file after | Hide comments
@@ -415,7 +415,7 class HFReader(ProcessingUnit):
415 415
416 416
417 417
418 def __searchFilesOffline(self,
418 def searchFilesOffLine(self,
419 419 path,
420 420 startDate,
421 421 endDate,
@@ -438,7 +438,7 class HFReader(ProcessingUnit):
438 438
439 439 return
440 440
441 def __searchFilesOnline(self,
441 def searchFilesOnLine(self,
442 442 path,
443 443 expLabel= "",
444 444 ext=None,
@@ -636,10 +636,10 class HFReader(ProcessingUnit):
636 636 if not(online):
637 637 print "Searching files in offline mode..."
638 638
639 self.__searchFilesOffline(path, startDate, endDate, ext, startTime, endTime, walk)
639 self.searchFilesOffLine(path, startDate, endDate, ext, startTime, endTime, walk)
640 640 else:
641 641 print "Searching files in online mode..."
642 self.__searchFilesOnline(path, walk,ext,set=set)
642 self.searchFilesOnLine(path, walk,ext,set=set)
643 643 if set==None:
644 644 pass
645 645 else:
@@ -647,7 +647,7 class HFReader(ProcessingUnit):
647 647
648 648 # for nTries in range(self.nTries):
649 649 #
650 # fullpath,file,year,month,day,set = self.__searchFilesOnline(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
650 # fullpath,file,year,month,day,set = self.searchFilesOnLine(path=path,expLabel=expLabel,ext=ext, walk=walk,set=set)
651 651 #
652 652 # if fullpath:
653 653 # break
Show file before | Show file after | Hide comments
@@ -106,9 +106,9 class AMISRReader(ProcessingUnit):
106 106 #self.findFiles()
107 107 if not(online):
108 108 #Busqueda de archivos offline
109 self.__searchFilesOffline(path, startDate, endDate, startTime, endTime, walk)
109 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
110 110 else:
111 self.__searchFilesOnline(path, startDate, endDate, startTime,endTime,walk)
111 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
112 112
113 113 if not(self.filenameList):
114 114 print "There is no files into the folder: %s"%(path)
@@ -329,7 +329,7 class AMISRReader(ProcessingUnit):
329 329 self.dirnameList = new_dirnameList
330 330 return 1
331 331
332 def __searchFilesOnline(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
332 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
333 333 endTime=datetime.time(23,59,59),walk=True):
334 334
335 335 if endDate ==None:
@@ -349,7 +349,7 class AMISRReader(ProcessingUnit):
349 349 return
350 350
351 351
352 def __searchFilesOffline(self,
352 def searchFilesOffLine(self,
353 353 path,
354 354 startDate,
355 355 endDate,
Show file before | Show file after | Hide comments
@@ -97,7 +97,7 class ParamReader(ProcessingUnit):
97 97 self.timezone = 'lt'
98 98
99 99 print "[Reading] Searching files in offline mode ..."
100 pathList, filenameList = self.__searchFilesOffLine(path, startDate=startDate, endDate=endDate,
100 pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
101 101 startTime=startTime, endTime=endTime,
102 102 ext=ext, walk=walk)
103 103
@@ -115,7 +115,7 class ParamReader(ProcessingUnit):
115 115
116 116 return
117 117
118 def __searchFilesOffLine(self,
118 def searchFilesOffLine(self,
119 119 path,
120 120 startDate=None,
121 121 endDate=None,
Show file before | Show file after | Hide comments
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now