##// END OF EJS Templates
corregido reorganización de apuntes en kamisr
joabAM -
r1420:78c29438e197
parent child
Show More
@@ -1,659 +1,660
1 ''''
1 ''''
2 Created on Set 9, 2015
2 Created on Set 9, 2015
3
3
4 @author: roj-idl71 Karim Kuyeng
4 @author: roj-idl71 Karim Kuyeng
5
5
6 @update: 2021, Joab Apaza
6 @update: 2021, Joab Apaza
7 '''
7 '''
8
8
9 import os
9 import os
10 import sys
10 import sys
11 import glob
11 import glob
12 import fnmatch
12 import fnmatch
13 import datetime
13 import datetime
14 import time
14 import time
15 import re
15 import re
16 import h5py
16 import h5py
17 import numpy
17 import numpy
18
18
19 try:
19 try:
20 from gevent import sleep
20 from gevent import sleep
21 except:
21 except:
22 from time import sleep
22 from time import sleep
23
23
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
24 from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
25 from schainpy.model.data.jrodata import Voltage
25 from schainpy.model.data.jrodata import Voltage
26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
26 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
27 from numpy import imag
27 from numpy import imag
28
28
29
29
30 class AMISRReader(ProcessingUnit):
30 class AMISRReader(ProcessingUnit):
31 '''
31 '''
32 classdocs
32 classdocs
33 '''
33 '''
34
34
35 def __init__(self):
35 def __init__(self):
36 '''
36 '''
37 Constructor
37 Constructor
38 '''
38 '''
39
39
40 ProcessingUnit.__init__(self)
40 ProcessingUnit.__init__(self)
41
41
42 self.set = None
42 self.set = None
43 self.subset = None
43 self.subset = None
44 self.extension_file = '.h5'
44 self.extension_file = '.h5'
45 self.dtc_str = 'dtc'
45 self.dtc_str = 'dtc'
46 self.dtc_id = 0
46 self.dtc_id = 0
47 self.status = True
47 self.status = True
48 self.isConfig = False
48 self.isConfig = False
49 self.dirnameList = []
49 self.dirnameList = []
50 self.filenameList = []
50 self.filenameList = []
51 self.fileIndex = None
51 self.fileIndex = None
52 self.flagNoMoreFiles = False
52 self.flagNoMoreFiles = False
53 self.flagIsNewFile = 0
53 self.flagIsNewFile = 0
54 self.filename = ''
54 self.filename = ''
55 self.amisrFilePointer = None
55 self.amisrFilePointer = None
56 self.realBeamCode = []
56 self.realBeamCode = []
57 self.beamCodeMap = None
57 self.beamCodeMap = None
58 self.azimuthList = []
58 self.azimuthList = []
59 self.elevationList = []
59 self.elevationList = []
60 self.dataShape = None
60 self.dataShape = None
61
61 self.flag_old_beams = False
62
62
63
63
64 self.profileIndex = 0
64 self.profileIndex = 0
65
65
66
66
67 self.beamCodeByFrame = None
67 self.beamCodeByFrame = None
68 self.radacTimeByFrame = None
68 self.radacTimeByFrame = None
69
69
70 self.dataset = None
70 self.dataset = None
71
71
72 self.__firstFile = True
72 self.__firstFile = True
73
73
74 self.buffer = None
74 self.buffer = None
75
75
76 self.timezone = 'ut'
76 self.timezone = 'ut'
77
77
78 self.__waitForNewFile = 20
78 self.__waitForNewFile = 20
79 self.__filename_online = None
79 self.__filename_online = None
80 #Is really necessary create the output object in the initializer
80 #Is really necessary create the output object in the initializer
81 self.dataOut = Voltage()
81 self.dataOut = Voltage()
82 self.dataOut.error=False
82 self.dataOut.error=False
83
83
84
84
85 def setup(self,path=None,
85 def setup(self,path=None,
86 startDate=None,
86 startDate=None,
87 endDate=None,
87 endDate=None,
88 startTime=None,
88 startTime=None,
89 endTime=None,
89 endTime=None,
90 walk=True,
90 walk=True,
91 timezone='ut',
91 timezone='ut',
92 all=0,
92 all=0,
93 code = None,
93 code = None,
94 nCode = 0,
94 nCode = 0,
95 nBaud = 0,
95 nBaud = 0,
96 online=False):
96 online=False
97 old_beams=False):
97
98
98
99
99
100
100 self.timezone = timezone
101 self.timezone = timezone
101 self.all = all
102 self.all = all
102 self.online = online
103 self.online = online
103
104 self.flag_old_beams = old_beams
104 self.code = code
105 self.code = code
105 self.nCode = int(nCode)
106 self.nCode = int(nCode)
106 self.nBaud = int(nBaud)
107 self.nBaud = int(nBaud)
107
108
108
109
109
110
110 #self.findFiles()
111 #self.findFiles()
111 if not(online):
112 if not(online):
112 #Busqueda de archivos offline
113 #Busqueda de archivos offline
113 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
114 self.searchFilesOffLine(path, startDate, endDate, startTime, endTime, walk)
114 else:
115 else:
115 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
116 self.searchFilesOnLine(path, startDate, endDate, startTime,endTime,walk)
116
117
117 if not(self.filenameList):
118 if not(self.filenameList):
118 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
119 raise schainpy.admin.SchainWarning("There is no files into the folder: %s"%(path))
119 sys.exit()
120 sys.exit()
120
121
121 self.fileIndex = 0
122 self.fileIndex = 0
122
123
123 self.readNextFile(online)
124 self.readNextFile(online)
124
125
125 '''
126 '''
126 Add code
127 Add code
127 '''
128 '''
128 self.isConfig = True
129 self.isConfig = True
129 # print("Setup Done")
130 # print("Setup Done")
130 pass
131 pass
131
132
132
133
133 def readAMISRHeader(self,fp):
134 def readAMISRHeader(self,fp):
134
135
135 if self.isConfig and (not self.flagNoMoreFiles):
136 if self.isConfig and (not self.flagNoMoreFiles):
136 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
137 newShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
137 if self.dataShape != newShape and newShape != None:
138 if self.dataShape != newShape and newShape != None:
138 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
139 raise schainpy.admin.SchainError("NEW FILE HAS A DIFFERENT SHAPE: ")
139 print(self.dataShape,newShape,"\n")
140 print(self.dataShape,newShape,"\n")
140 return 0
141 return 0
141 else:
142 else:
142 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
143 self.dataShape = fp.get('Raw11/Data/Samples/Data').shape[1:]
143
144
144
145
145 header = 'Raw11/Data/RadacHeader'
146 header = 'Raw11/Data/RadacHeader'
146 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
147 self.beamCodeByPulse = fp.get(header+'/BeamCode') # LIST OF BEAMS PER PROFILE, TO BE USED ON REARRANGE
147 if (self.startDate> datetime.date(2021, 7, 15)): #Se cambió la forma de extracción de Apuntes el 17
148 if (self.startDate> datetime.date(2021, 7, 15)) or self.flag_old_beams: #Se cambió la forma de extracción de Apuntes el 17 o forzar con flag de reorganización
148 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
149 self.beamcodeFile = fp['Setup/Beamcodefile'][()].decode()
149 self.trueBeams = self.beamcodeFile.split("\n")
150 self.trueBeams = self.beamcodeFile.split("\n")
150 self.trueBeams.pop()#remove last
151 self.trueBeams.pop()#remove last
151 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
152 [self.realBeamCode.append(x) for x in self.trueBeams if x not in self.realBeamCode]
152 self.beamCode = [int(x, 16) for x in self.realBeamCode]
153 self.beamCode = [int(x, 16) for x in self.realBeamCode]
153 else:
154 else:
154 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
155 _beamCode= fp.get('Raw11/Data/Beamcodes') #se usa la manera previa al cambio de apuntes
155 self.beamCode = _beamCode[0,:]
156 self.beamCode = _beamCode[0,:]
156
157
157 if self.beamCodeMap == None:
158 if self.beamCodeMap == None:
158 self.beamCodeMap = fp['Setup/BeamcodeMap']
159 self.beamCodeMap = fp['Setup/BeamcodeMap']
159 for beam in self.beamCode:
160 for beam in self.beamCode:
160 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
161 beamAziElev = numpy.where(self.beamCodeMap[:,0]==beam)
161 beamAziElev = beamAziElev[0].squeeze()
162 beamAziElev = beamAziElev[0].squeeze()
162 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
163 self.azimuthList.append(self.beamCodeMap[beamAziElev,1])
163 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
164 self.elevationList.append(self.beamCodeMap[beamAziElev,2])
164 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
165 #print("Beamssss: ",self.beamCodeMap[beamAziElev,1],self.beamCodeMap[beamAziElev,2])
165 #print(self.beamCode)
166 #print(self.beamCode)
166 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
167 #self.code = fp.get(header+'/Code') # NOT USE FOR THIS
167 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
168 self.frameCount = fp.get(header+'/FrameCount')# NOT USE FOR THIS
168 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
169 self.modeGroup = fp.get(header+'/ModeGroup')# NOT USE FOR THIS
169 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
170 self.nsamplesPulse = fp.get(header+'/NSamplesPulse')# TO GET NSA OR USING DATA FOR THAT
170 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
171 self.pulseCount = fp.get(header+'/PulseCount')# NOT USE FOR THIS
171 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
172 self.radacTime = fp.get(header+'/RadacTime')# 1st TIME ON FILE ANDE CALCULATE THE REST WITH IPP*nindexprofile
172 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
173 self.timeCount = fp.get(header+'/TimeCount')# NOT USE FOR THIS
173 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
174 self.timeStatus = fp.get(header+'/TimeStatus')# NOT USE FOR THIS
174 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
175 self.rangeFromFile = fp.get('Raw11/Data/Samples/Range')
175 self.frequency = fp.get('Rx/Frequency')
176 self.frequency = fp.get('Rx/Frequency')
176 txAus = fp.get('Raw11/Data/Pulsewidth')
177 txAus = fp.get('Raw11/Data/Pulsewidth')
177
178
178
179
179 self.nblocks = self.pulseCount.shape[0] #nblocks
180 self.nblocks = self.pulseCount.shape[0] #nblocks
180
181
181 self.nprofiles = self.pulseCount.shape[1] #nprofile
182 self.nprofiles = self.pulseCount.shape[1] #nprofile
182 self.nsa = self.nsamplesPulse[0,0] #ngates
183 self.nsa = self.nsamplesPulse[0,0] #ngates
183 self.nchannels = len(self.beamCode)
184 self.nchannels = len(self.beamCode)
184 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
185 self.ippSeconds = (self.radacTime[0][1] -self.radacTime[0][0]) #Ipp in seconds
185 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
186 #self.__waitForNewFile = self.nblocks # wait depending on the number of blocks since each block is 1 sec
186 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
187 self.__waitForNewFile = self.nblocks * self.nprofiles * self.ippSeconds # wait until new file is created
187
188
188 #filling radar controller header parameters
189 #filling radar controller header parameters
189 self.__ippKm = self.ippSeconds *.15*1e6 # in km
190 self.__ippKm = self.ippSeconds *.15*1e6 # in km
190 self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
191 self.__txA = (txAus.value)*.15 #(ipp[us]*.15km/1us) in km
191 self.__txB = 0
192 self.__txB = 0
192 nWindows=1
193 nWindows=1
193 self.__nSamples = self.nsa
194 self.__nSamples = self.nsa
194 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
195 self.__firstHeight = self.rangeFromFile[0][0]/1000 #in km
195 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
196 self.__deltaHeight = (self.rangeFromFile[0][1] - self.rangeFromFile[0][0])/1000
196
197
197 #for now until understand why the code saved is different (code included even though code not in tuf file)
198 #for now until understand why the code saved is different (code included even though code not in tuf file)
198 #self.__codeType = 0
199 #self.__codeType = 0
199 # self.__nCode = None
200 # self.__nCode = None
200 # self.__nBaud = None
201 # self.__nBaud = None
201 self.__code = self.code
202 self.__code = self.code
202 self.__codeType = 0
203 self.__codeType = 0
203 if self.code != None:
204 if self.code != None:
204 self.__codeType = 1
205 self.__codeType = 1
205 self.__nCode = self.nCode
206 self.__nCode = self.nCode
206 self.__nBaud = self.nBaud
207 self.__nBaud = self.nBaud
207 #self.__code = 0
208 #self.__code = 0
208
209
209 #filling system header parameters
210 #filling system header parameters
210 self.__nSamples = self.nsa
211 self.__nSamples = self.nsa
211 self.newProfiles = self.nprofiles/self.nchannels
212 self.newProfiles = self.nprofiles/self.nchannels
212 self.__channelList = list(range(self.nchannels))
213 self.__channelList = list(range(self.nchannels))
213
214
214 self.__frequency = self.frequency[0][0]
215 self.__frequency = self.frequency[0][0]
215
216
216
217
217 return 1
218 return 1
218
219
219
220
220 def createBuffers(self):
221 def createBuffers(self):
221
222
222 pass
223 pass
223
224
224 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
225 def __setParameters(self,path='', startDate='',endDate='',startTime='', endTime='', walk=''):
225 self.path = path
226 self.path = path
226 self.startDate = startDate
227 self.startDate = startDate
227 self.endDate = endDate
228 self.endDate = endDate
228 self.startTime = startTime
229 self.startTime = startTime
229 self.endTime = endTime
230 self.endTime = endTime
230 self.walk = walk
231 self.walk = walk
231
232
232 def __checkPath(self):
233 def __checkPath(self):
233 if os.path.exists(self.path):
234 if os.path.exists(self.path):
234 self.status = 1
235 self.status = 1
235 else:
236 else:
236 self.status = 0
237 self.status = 0
237 print('Path:%s does not exists'%self.path)
238 print('Path:%s does not exists'%self.path)
238
239
239 return
240 return
240
241
241
242
242 def __selDates(self, amisr_dirname_format):
243 def __selDates(self, amisr_dirname_format):
243 try:
244 try:
244 year = int(amisr_dirname_format[0:4])
245 year = int(amisr_dirname_format[0:4])
245 month = int(amisr_dirname_format[4:6])
246 month = int(amisr_dirname_format[4:6])
246 dom = int(amisr_dirname_format[6:8])
247 dom = int(amisr_dirname_format[6:8])
247 thisDate = datetime.date(year,month,dom)
248 thisDate = datetime.date(year,month,dom)
248 #margen de un día extra, igual luego se filtra for fecha y hora
249 #margen de un día extra, igual luego se filtra for fecha y hora
249 if (thisDate>=(self.startDate - datetime.timedelta(days=1)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
250 if (thisDate>=(self.startDate - datetime.timedelta(days=1)) and thisDate <= (self.endDate)+ datetime.timedelta(days=1)):
250 return amisr_dirname_format
251 return amisr_dirname_format
251 except:
252 except:
252 return None
253 return None
253
254
254
255
255 def __findDataForDates(self,online=False):
256 def __findDataForDates(self,online=False):
256
257
257 if not(self.status):
258 if not(self.status):
258 return None
259 return None
259
260
260 pat = '\d+.\d+'
261 pat = '\d+.\d+'
261 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
262 dirnameList = [re.search(pat,x) for x in os.listdir(self.path)]
262 dirnameList = [x for x in dirnameList if x!=None]
263 dirnameList = [x for x in dirnameList if x!=None]
263 dirnameList = [x.string for x in dirnameList]
264 dirnameList = [x.string for x in dirnameList]
264 if not(online):
265 if not(online):
265 dirnameList = [self.__selDates(x) for x in dirnameList]
266 dirnameList = [self.__selDates(x) for x in dirnameList]
266 dirnameList = [x for x in dirnameList if x!=None]
267 dirnameList = [x for x in dirnameList if x!=None]
267 if len(dirnameList)>0:
268 if len(dirnameList)>0:
268 self.status = 1
269 self.status = 1
269 self.dirnameList = dirnameList
270 self.dirnameList = dirnameList
270 self.dirnameList.sort()
271 self.dirnameList.sort()
271 else:
272 else:
272 self.status = 0
273 self.status = 0
273 return None
274 return None
274
275
275 def __getTimeFromData(self):
276 def __getTimeFromData(self):
276 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
277 startDateTime_Reader = datetime.datetime.combine(self.startDate,self.startTime)
277 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
278 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
278
279
279 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
280 print('Filtering Files from %s to %s'%(startDateTime_Reader, endDateTime_Reader))
280 print('........................................')
281 print('........................................')
281 filter_filenameList = []
282 filter_filenameList = []
282 self.filenameList.sort()
283 self.filenameList.sort()
283 #for i in range(len(self.filenameList)-1):
284 #for i in range(len(self.filenameList)-1):
284 for i in range(len(self.filenameList)):
285 for i in range(len(self.filenameList)):
285 filename = self.filenameList[i]
286 filename = self.filenameList[i]
286 fp = h5py.File(filename,'r')
287 fp = h5py.File(filename,'r')
287 time_str = fp.get('Time/RadacTimeString')
288 time_str = fp.get('Time/RadacTimeString')
288
289
289 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
290 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
290 #startDateTimeStr_File = "2019-12-16 09:21:11"
291 #startDateTimeStr_File = "2019-12-16 09:21:11"
291 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
292 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
292 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
293 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
293
294
294 #endDateTimeStr_File = "2019-12-16 11:10:11"
295 #endDateTimeStr_File = "2019-12-16 11:10:11"
295 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
296 endDateTimeStr_File = time_str[-1][-1].decode('UTF-8').split('.')[0]
296 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
297 junk = time.strptime(endDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
297 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
298 endDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
298
299
299 fp.close()
300 fp.close()
300
301
301 #print("check time", startDateTime_File)
302 #print("check time", startDateTime_File)
302 if self.timezone == 'lt':
303 if self.timezone == 'lt':
303 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
304 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
304 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
305 endDateTime_File = endDateTime_File - datetime.timedelta(minutes = 300)
305 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
306 if (startDateTime_File >=startDateTime_Reader and endDateTime_File<=endDateTime_Reader):
306 filter_filenameList.append(filename)
307 filter_filenameList.append(filename)
307
308
308 if (startDateTime_File>endDateTime_Reader):
309 if (startDateTime_File>endDateTime_Reader):
309 break
310 break
310
311
311
312
312 filter_filenameList.sort()
313 filter_filenameList.sort()
313 self.filenameList = filter_filenameList
314 self.filenameList = filter_filenameList
314
315
315 return 1
316 return 1
316
317
317 def __filterByGlob1(self, dirName):
318 def __filterByGlob1(self, dirName):
318 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
319 filter_files = glob.glob1(dirName, '*.*%s'%self.extension_file)
319 filter_files.sort()
320 filter_files.sort()
320 filterDict = {}
321 filterDict = {}
321 filterDict.setdefault(dirName)
322 filterDict.setdefault(dirName)
322 filterDict[dirName] = filter_files
323 filterDict[dirName] = filter_files
323 return filterDict
324 return filterDict
324
325
325 def __getFilenameList(self, fileListInKeys, dirList):
326 def __getFilenameList(self, fileListInKeys, dirList):
326 for value in fileListInKeys:
327 for value in fileListInKeys:
327 dirName = list(value.keys())[0]
328 dirName = list(value.keys())[0]
328 for file in value[dirName]:
329 for file in value[dirName]:
329 filename = os.path.join(dirName, file)
330 filename = os.path.join(dirName, file)
330 self.filenameList.append(filename)
331 self.filenameList.append(filename)
331
332
332
333
333 def __selectDataForTimes(self, online=False):
334 def __selectDataForTimes(self, online=False):
334 #aun no esta implementado el filtro for tiempo
335 #aun no esta implementado el filtro for tiempo
335 if not(self.status):
336 if not(self.status):
336 return None
337 return None
337
338
338 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
339 dirList = [os.path.join(self.path,x) for x in self.dirnameList]
339 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
340 fileListInKeys = [self.__filterByGlob1(x) for x in dirList]
340 self.__getFilenameList(fileListInKeys, dirList)
341 self.__getFilenameList(fileListInKeys, dirList)
341 if not(online):
342 if not(online):
342 #filtro por tiempo
343 #filtro por tiempo
343 if not(self.all):
344 if not(self.all):
344 self.__getTimeFromData()
345 self.__getTimeFromData()
345
346
346 if len(self.filenameList)>0:
347 if len(self.filenameList)>0:
347 self.status = 1
348 self.status = 1
348 self.filenameList.sort()
349 self.filenameList.sort()
349 else:
350 else:
350 self.status = 0
351 self.status = 0
351 return None
352 return None
352
353
353 else:
354 else:
354 #get the last file - 1
355 #get the last file - 1
355 self.filenameList = [self.filenameList[-2]]
356 self.filenameList = [self.filenameList[-2]]
356 new_dirnameList = []
357 new_dirnameList = []
357 for dirname in self.dirnameList:
358 for dirname in self.dirnameList:
358 junk = numpy.array([dirname in x for x in self.filenameList])
359 junk = numpy.array([dirname in x for x in self.filenameList])
359 junk_sum = junk.sum()
360 junk_sum = junk.sum()
360 if junk_sum > 0:
361 if junk_sum > 0:
361 new_dirnameList.append(dirname)
362 new_dirnameList.append(dirname)
362 self.dirnameList = new_dirnameList
363 self.dirnameList = new_dirnameList
363 return 1
364 return 1
364
365
365 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
366 def searchFilesOnLine(self, path, startDate, endDate, startTime=datetime.time(0,0,0),
366 endTime=datetime.time(23,59,59),walk=True):
367 endTime=datetime.time(23,59,59),walk=True):
367
368
368 if endDate ==None:
369 if endDate ==None:
369 startDate = datetime.datetime.utcnow().date()
370 startDate = datetime.datetime.utcnow().date()
370 endDate = datetime.datetime.utcnow().date()
371 endDate = datetime.datetime.utcnow().date()
371
372
372 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
373 self.__setParameters(path=path, startDate=startDate, endDate=endDate,startTime = startTime,endTime=endTime, walk=walk)
373
374
374 self.__checkPath()
375 self.__checkPath()
375
376
376 self.__findDataForDates(online=True)
377 self.__findDataForDates(online=True)
377
378
378 self.dirnameList = [self.dirnameList[-1]]
379 self.dirnameList = [self.dirnameList[-1]]
379
380
380 self.__selectDataForTimes(online=True)
381 self.__selectDataForTimes(online=True)
381
382
382 return
383 return
383
384
384
385
385 def searchFilesOffLine(self,
386 def searchFilesOffLine(self,
386 path,
387 path,
387 startDate,
388 startDate,
388 endDate,
389 endDate,
389 startTime=datetime.time(0,0,0),
390 startTime=datetime.time(0,0,0),
390 endTime=datetime.time(23,59,59),
391 endTime=datetime.time(23,59,59),
391 walk=True):
392 walk=True):
392
393
393 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
394 self.__setParameters(path, startDate, endDate, startTime, endTime, walk)
394
395
395 self.__checkPath()
396 self.__checkPath()
396
397
397 self.__findDataForDates()
398 self.__findDataForDates()
398
399
399 self.__selectDataForTimes()
400 self.__selectDataForTimes()
400
401
401 for i in range(len(self.filenameList)):
402 for i in range(len(self.filenameList)):
402 print("%s" %(self.filenameList[i]))
403 print("%s" %(self.filenameList[i]))
403
404
404 return
405 return
405
406
406 def __setNextFileOffline(self):
407 def __setNextFileOffline(self):
407
408
408 try:
409 try:
409 self.filename = self.filenameList[self.fileIndex]
410 self.filename = self.filenameList[self.fileIndex]
410 self.amisrFilePointer = h5py.File(self.filename,'r')
411 self.amisrFilePointer = h5py.File(self.filename,'r')
411 self.fileIndex += 1
412 self.fileIndex += 1
412 except:
413 except:
413 self.flagNoMoreFiles = 1
414 self.flagNoMoreFiles = 1
414 raise schainpy.admin.SchainError('No more files to read')
415 raise schainpy.admin.SchainError('No more files to read')
415 return 0
416 return 0
416
417
417 self.flagIsNewFile = 1
418 self.flagIsNewFile = 1
418 print("Setting the file: %s"%self.filename)
419 print("Setting the file: %s"%self.filename)
419
420
420 return 1
421 return 1
421
422
422
423
423 def __setNextFileOnline(self):
424 def __setNextFileOnline(self):
424 filename = self.filenameList[0]
425 filename = self.filenameList[0]
425 if self.__filename_online != None:
426 if self.__filename_online != None:
426 self.__selectDataForTimes(online=True)
427 self.__selectDataForTimes(online=True)
427 filename = self.filenameList[0]
428 filename = self.filenameList[0]
428 wait = 0
429 wait = 0
429 self.__waitForNewFile=300 ## DEBUG:
430 self.__waitForNewFile=300 ## DEBUG:
430 while self.__filename_online == filename:
431 while self.__filename_online == filename:
431 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
432 print('waiting %d seconds to get a new file...'%(self.__waitForNewFile))
432 if wait == 5:
433 if wait == 5:
433 self.flagNoMoreFiles = 1
434 self.flagNoMoreFiles = 1
434 return 0
435 return 0
435 sleep(self.__waitForNewFile)
436 sleep(self.__waitForNewFile)
436 self.__selectDataForTimes(online=True)
437 self.__selectDataForTimes(online=True)
437 filename = self.filenameList[0]
438 filename = self.filenameList[0]
438 wait += 1
439 wait += 1
439
440
440 self.__filename_online = filename
441 self.__filename_online = filename
441
442
442 self.amisrFilePointer = h5py.File(filename,'r')
443 self.amisrFilePointer = h5py.File(filename,'r')
443 self.flagIsNewFile = 1
444 self.flagIsNewFile = 1
444 self.filename = filename
445 self.filename = filename
445 print("Setting the file: %s"%self.filename)
446 print("Setting the file: %s"%self.filename)
446 return 1
447 return 1
447
448
448
449
449 def readData(self):
450 def readData(self):
450 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
451 buffer = self.amisrFilePointer.get('Raw11/Data/Samples/Data')
451 re = buffer[:,:,:,0]
452 re = buffer[:,:,:,0]
452 im = buffer[:,:,:,1]
453 im = buffer[:,:,:,1]
453 dataset = re + im*1j
454 dataset = re + im*1j
454
455
455 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
456 self.radacTime = self.amisrFilePointer.get('Raw11/Data/RadacHeader/RadacTime')
456 timeset = self.radacTime[:,0]
457 timeset = self.radacTime[:,0]
457
458
458 return dataset,timeset
459 return dataset,timeset
459
460
460 def reshapeData(self):
461 def reshapeData(self):
461 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
462 #self.beamCodeByPulse, self.beamCode, self.nblocks, self.nprofiles, self.nsa,
462 channels = self.beamCodeByPulse[0,:]
463 channels = self.beamCodeByPulse[0,:]
463 nchan = self.nchannels
464 nchan = self.nchannels
464 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
465 #self.newProfiles = self.nprofiles/nchan #must be defined on filljroheader
465 nblocks = self.nblocks
466 nblocks = self.nblocks
466 nsamples = self.nsa
467 nsamples = self.nsa
467
468
468 #Dimensions : nChannels, nProfiles, nSamples
469 #Dimensions : nChannels, nProfiles, nSamples
469 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
470 new_block = numpy.empty((nblocks, nchan, numpy.int_(self.newProfiles), nsamples), dtype="complex64")
470 ############################################
471 ############################################
471
472
472 for thisChannel in range(nchan):
473 for thisChannel in range(nchan):
473 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
474 new_block[:,thisChannel,:,:] = self.dataset[:,numpy.where(channels==self.beamCode[thisChannel])[0],:]
474
475
475
476
476 new_block = numpy.transpose(new_block, (1,0,2,3))
477 new_block = numpy.transpose(new_block, (1,0,2,3))
477 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
478 new_block = numpy.reshape(new_block, (nchan,-1, nsamples))
478
479
479 return new_block
480 return new_block
480
481
481 def updateIndexes(self):
482 def updateIndexes(self):
482
483
483 pass
484 pass
484
485
485 def fillJROHeader(self):
486 def fillJROHeader(self):
486
487
487 #fill radar controller header
488 #fill radar controller header
488 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
489 self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ipp=self.__ippKm,
489 txA=self.__txA,
490 txA=self.__txA,
490 txB=0,
491 txB=0,
491 nWindows=1,
492 nWindows=1,
492 nHeights=self.__nSamples,
493 nHeights=self.__nSamples,
493 firstHeight=self.__firstHeight,
494 firstHeight=self.__firstHeight,
494 deltaHeight=self.__deltaHeight,
495 deltaHeight=self.__deltaHeight,
495 codeType=self.__codeType,
496 codeType=self.__codeType,
496 nCode=self.__nCode, nBaud=self.__nBaud,
497 nCode=self.__nCode, nBaud=self.__nBaud,
497 code = self.__code,
498 code = self.__code,
498 fClock=1)
499 fClock=1)
499
500
500 #fill system header
501 #fill system header
501 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
502 self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
502 nProfiles=self.newProfiles,
503 nProfiles=self.newProfiles,
503 nChannels=len(self.__channelList),
504 nChannels=len(self.__channelList),
504 adcResolution=14,
505 adcResolution=14,
505 pciDioBusWidth=32)
506 pciDioBusWidth=32)
506
507
507 self.dataOut.type = "Voltage"
508 self.dataOut.type = "Voltage"
508 self.dataOut.data = None
509 self.dataOut.data = None
509 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
510 self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
510 # self.dataOut.nChannels = 0
511 # self.dataOut.nChannels = 0
511
512
512 # self.dataOut.nHeights = 0
513 # self.dataOut.nHeights = 0
513
514
514 self.dataOut.nProfiles = self.newProfiles*self.nblocks
515 self.dataOut.nProfiles = self.newProfiles*self.nblocks
515 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
516 #self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
516 ranges = numpy.reshape(self.rangeFromFile.value,(-1))
517 ranges = numpy.reshape(self.rangeFromFile.value,(-1))
517 self.dataOut.heightList = ranges/1000.0 #km
518 self.dataOut.heightList = ranges/1000.0 #km
518 self.dataOut.channelList = self.__channelList
519 self.dataOut.channelList = self.__channelList
519 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
520 self.dataOut.blocksize = self.dataOut.nChannels * self.dataOut.nHeights
520
521
521 # self.dataOut.channelIndexList = None
522 # self.dataOut.channelIndexList = None
522
523
523
524
524 self.dataOut.azimuthList = numpy.array(self.azimuthList)
525 self.dataOut.azimuthList = numpy.array(self.azimuthList)
525 self.dataOut.elevationList = numpy.array(self.elevationList)
526 self.dataOut.elevationList = numpy.array(self.elevationList)
526 self.dataOut.codeList = numpy.array(self.beamCode)
527 self.dataOut.codeList = numpy.array(self.beamCode)
527 #print(self.dataOut.elevationList)
528 #print(self.dataOut.elevationList)
528 self.dataOut.flagNoData = True
529 self.dataOut.flagNoData = True
529
530
530 #Set to TRUE if the data is discontinuous
531 #Set to TRUE if the data is discontinuous
531 self.dataOut.flagDiscontinuousBlock = False
532 self.dataOut.flagDiscontinuousBlock = False
532
533
533 self.dataOut.utctime = None
534 self.dataOut.utctime = None
534
535
535 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
536 #self.dataOut.timeZone = -5 #self.__timezone/60 #timezone like jroheader, difference in minutes between UTC and localtime
536 if self.timezone == 'lt':
537 if self.timezone == 'lt':
537 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
538 self.dataOut.timeZone = time.timezone / 60. #get the timezone in minutes
538 else:
539 else:
539 self.dataOut.timeZone = 0 #by default time is UTC
540 self.dataOut.timeZone = 0 #by default time is UTC
540
541
541 self.dataOut.dstFlag = 0
542 self.dataOut.dstFlag = 0
542 self.dataOut.errorCount = 0
543 self.dataOut.errorCount = 0
543 self.dataOut.nCohInt = 1
544 self.dataOut.nCohInt = 1
544 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
545 self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
545 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
546 self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
546 self.dataOut.flagShiftFFT = False
547 self.dataOut.flagShiftFFT = False
547 self.dataOut.ippSeconds = self.ippSeconds
548 self.dataOut.ippSeconds = self.ippSeconds
548
549
549 #Time interval between profiles
550 #Time interval between profiles
550 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
551 #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
551
552
552 self.dataOut.frequency = self.__frequency
553 self.dataOut.frequency = self.__frequency
553 self.dataOut.realtime = self.online
554 self.dataOut.realtime = self.online
554 pass
555 pass
555
556
556 def readNextFile(self,online=False):
557 def readNextFile(self,online=False):
557
558
558 if not(online):
559 if not(online):
559 newFile = self.__setNextFileOffline()
560 newFile = self.__setNextFileOffline()
560 else:
561 else:
561 newFile = self.__setNextFileOnline()
562 newFile = self.__setNextFileOnline()
562
563
563 if not(newFile):
564 if not(newFile):
564 self.dataOut.error = True
565 self.dataOut.error = True
565 return 0
566 return 0
566
567
567 if not self.readAMISRHeader(self.amisrFilePointer):
568 if not self.readAMISRHeader(self.amisrFilePointer):
568 self.dataOut.error = True
569 self.dataOut.error = True
569 return 0
570 return 0
570
571
571 self.createBuffers()
572 self.createBuffers()
572 self.fillJROHeader()
573 self.fillJROHeader()
573
574
574 #self.__firstFile = False
575 #self.__firstFile = False
575
576
576
577
577
578
578 self.dataset,self.timeset = self.readData()
579 self.dataset,self.timeset = self.readData()
579
580
580 if self.endDate!=None:
581 if self.endDate!=None:
581 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
582 endDateTime_Reader = datetime.datetime.combine(self.endDate,self.endTime)
582 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
583 time_str = self.amisrFilePointer.get('Time/RadacTimeString')
583 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
584 startDateTimeStr_File = time_str[0][0].decode('UTF-8').split('.')[0]
584 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
585 junk = time.strptime(startDateTimeStr_File, '%Y-%m-%d %H:%M:%S')
585 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
586 startDateTime_File = datetime.datetime(junk.tm_year,junk.tm_mon,junk.tm_mday,junk.tm_hour, junk.tm_min, junk.tm_sec)
586 if self.timezone == 'lt':
587 if self.timezone == 'lt':
587 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
588 startDateTime_File = startDateTime_File - datetime.timedelta(minutes = 300)
588 if (startDateTime_File>endDateTime_Reader):
589 if (startDateTime_File>endDateTime_Reader):
589 return 0
590 return 0
590
591
591 self.jrodataset = self.reshapeData()
592 self.jrodataset = self.reshapeData()
592 #----self.updateIndexes()
593 #----self.updateIndexes()
593 self.profileIndex = 0
594 self.profileIndex = 0
594
595
595 return 1
596 return 1
596
597
597
598
598 def __hasNotDataInBuffer(self):
599 def __hasNotDataInBuffer(self):
599 if self.profileIndex >= (self.newProfiles*self.nblocks):
600 if self.profileIndex >= (self.newProfiles*self.nblocks):
600 return 1
601 return 1
601 return 0
602 return 0
602
603
603
604
604 def getData(self):
605 def getData(self):
605
606
606 if self.flagNoMoreFiles:
607 if self.flagNoMoreFiles:
607 self.dataOut.flagNoData = True
608 self.dataOut.flagNoData = True
608 return 0
609 return 0
609
610
610 if self.__hasNotDataInBuffer():
611 if self.__hasNotDataInBuffer():
611 if not (self.readNextFile(self.online)):
612 if not (self.readNextFile(self.online)):
612 return 0
613 return 0
613
614
614
615
615 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
616 if self.dataset is None: # setear esta condicion cuando no hayan datos por leer
616 self.dataOut.flagNoData = True
617 self.dataOut.flagNoData = True
617 return 0
618 return 0
618
619
619 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
620 #self.dataOut.data = numpy.reshape(self.jrodataset[self.profileIndex,:],(1,-1))
620
621
621 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
622 self.dataOut.data = self.jrodataset[:,self.profileIndex,:]
622
623
623 #print("R_t",self.timeset)
624 #print("R_t",self.timeset)
624
625
625 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
626 #self.dataOut.utctime = self.jrotimeset[self.profileIndex]
626 #verificar basic header de jro data y ver si es compatible con este valor
627 #verificar basic header de jro data y ver si es compatible con este valor
627 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
628 #self.dataOut.utctime = self.timeset + (self.profileIndex * self.ippSeconds * self.nchannels)
628 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
629 indexprof = numpy.mod(self.profileIndex, self.newProfiles)
629 indexblock = self.profileIndex/self.newProfiles
630 indexblock = self.profileIndex/self.newProfiles
630 #print (indexblock, indexprof)
631 #print (indexblock, indexprof)
631 diffUTC = 0
632 diffUTC = 0
632 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
633 t_comp = (indexprof * self.ippSeconds * self.nchannels) + diffUTC #
633
634
634 #print("utc :",indexblock," __ ",t_comp)
635 #print("utc :",indexblock," __ ",t_comp)
635 #print(numpy.shape(self.timeset))
636 #print(numpy.shape(self.timeset))
636 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
637 self.dataOut.utctime = self.timeset[numpy.int_(indexblock)] + t_comp
637 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
638 #self.dataOut.utctime = self.timeset[self.profileIndex] + t_comp
638
639
639 self.dataOut.profileIndex = self.profileIndex
640 self.dataOut.profileIndex = self.profileIndex
640 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
641 #print("N profile:",self.profileIndex,self.newProfiles,self.nblocks,self.dataOut.utctime)
641 self.dataOut.flagNoData = False
642 self.dataOut.flagNoData = False
642 # if indexprof == 0:
643 # if indexprof == 0:
643 # print("kamisr: ",self.dataOut.utctime)
644 # print("kamisr: ",self.dataOut.utctime)
644
645
645 self.profileIndex += 1
646 self.profileIndex += 1
646
647
647 return self.dataOut.data #retorno necesario??
648 return self.dataOut.data #retorno necesario??
648
649
649
650
650 def run(self, **kwargs):
651 def run(self, **kwargs):
651 '''
652 '''
652 This method will be called many times so here you should put all your code
653 This method will be called many times so here you should put all your code
653 '''
654 '''
654 #print("running kamisr")
655 #print("running kamisr")
655 if not self.isConfig:
656 if not self.isConfig:
656 self.setup(**kwargs)
657 self.setup(**kwargs)
657 self.isConfig = True
658 self.isConfig = True
658
659
659 self.getData()
660 self.getData()
@@ -1,1113 +1,1112
1 '''
1 '''
2 @author: Daniel Suarez
2 @author: Daniel Suarez
3 '''
3 '''
4 import os
4 import os
5 import glob
5 import glob
6 import ftplib
6 import ftplib
7
7
8 try:
8 try:
9 import paramiko
9 import paramiko
10 import scp
10 import scp
11 except:
11 except:
12 pass
12 pass
13
13
14 import time
14 import time
15
15
16 import threading
16 import threading
17 Thread = threading.Thread
17 Thread = threading.Thread
18
18
19 # try:
19 # try:
20 # from gevent import sleep
20 # from gevent import sleep
21 # except:
21 # except:
22 from time import sleep
22 from time import sleep
23 from schainpy.model.data.jrodata import *
23 from schainpy.model.data.jrodata import *
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
24 from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
25 #@MPDecorator
25 #@MPDecorator
26 class Remote(Thread):
26 class Remote(Thread):
27 """
27 """
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
28 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
29 used to upload or download files remotely.
29 used to upload or download files remotely.
30
30
31 Non-standard Python modules used:
31 Non-standard Python modules used:
32 None
32 None
33
33
34 Written by:
34 Written by:
35 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
35 "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
36 Modified by:
36 Modified by:
37 -
37 -
38 """
38 """
39
39
40 server = None
40 server = None
41 username = None
41 username = None
42 password = None
42 password = None
43 remotefolder = None
43 remotefolder = None
44 key_filename=None
44 key_filename=None
45
45
46 period = 60
46 period = 60
47 fileList = []
47 fileList = []
48 bussy = False
48 bussy = False
49
49
50 def __init__(self, server, username, password, remotefolder, period=60,key_filename=None):
50 def __init__(self, server, username, password, remotefolder, period=60,key_filename=None):
51
51
52 Thread.__init__(self)
52 Thread.__init__(self)
53
53
54 self.setDaemon(True)
54 self.setDaemon(True)
55
55
56 self.status = 0
56 self.status = 0
57
57
58 self.__server = server
58 self.__server = server
59 self.__username = username
59 self.__username = username
60 self.__password = password
60 self.__password = password
61 self.__remotefolder = remotefolder
61 self.__remotefolder = remotefolder
62
62
63 self.period = period
63 self.period = period
64 self.key_filename = key_filename
64 self.key_filename = key_filename
65 self.fileList = []
65 self.fileList = []
66 self.bussy = False
66 self.bussy = False
67
67
68 self.stopFlag = False
68 self.stopFlag = False
69
69
70 print("[Remote Server] Opening server: %s" %self.__server)
70 print("[Remote Server] Opening server: %s" %self.__server)
71 if self.open(self.__server, self.__username, self.__password, self.__remotefolder,key_filename=self.key_filename):
71 if self.open(self.__server, self.__username, self.__password, self.__remotefolder,key_filename=self.key_filename):
72 print("[Remote Server] %s server was opened successfully" %self.__server)
72 print("[Remote Server] %s server was opened successfully" %self.__server)
73
73
74 #self.close()
74 #self.close()
75
75
76 self.mutex = threading.Lock()
76 self.mutex = threading.Lock()
77
77
78 def stop(self):
78 def stop(self):
79
79
80 self.stopFlag = True
80 self.stopFlag = True
81 self.join(10)
81 self.join(10)
82
82
83 def open(self):
83 def open(self):
84 """
84 """
85 Connect to server and create a connection class (FTP or SSH) to remote server.
85 Connect to server and create a connection class (FTP or SSH) to remote server.
86 """
86 """
87 raise NotImplementedError("Implement this method in child class")
87 raise NotImplementedError("Implement this method in child class")
88
88
89 def close(self):
89 def close(self):
90 """
90 """
91 Close connection to server
91 Close connection to server
92 """
92 """
93 raise NotImplementedError("Implement this method in child class")
93 raise NotImplementedError("Implement this method in child class")
94
94
95 def mkdir(self, remotefolder):
95 def mkdir(self, remotefolder):
96 """
96 """
97 Create a folder remotely
97 Create a folder remotely
98 """
98 """
99 raise NotImplementedError("Implement this method in child class")
99 raise NotImplementedError("Implement this method in child class")
100
100
101 def cd(self, remotefolder):
101 def cd(self, remotefolder):
102 """
102 """
103 Change working directory in remote server
103 Change working directory in remote server
104 """
104 """
105 raise NotImplementedError("Implement this method in child class")
105 raise NotImplementedError("Implement this method in child class")
106
106
107 def download(self, filename, localfolder=None):
107 def download(self, filename, localfolder=None):
108 """
108 """
109 Download a file from server to local host
109 Download a file from server to local host
110 """
110 """
111 raise NotImplementedError("Implement this method in child class")
111 raise NotImplementedError("Implement this method in child class")
112
112
113 def sendFile(self, fullfilename):
113 def sendFile(self, fullfilename):
114 """
114 """
115 sendFile method is used to upload a local file to the current directory in remote server
115 sendFile method is used to upload a local file to the current directory in remote server
116
116
117 Inputs:
117 Inputs:
118 fullfilename - full path name of local file to store in remote directory
118 fullfilename - full path name of local file to store in remote directory
119
119
120 Returns:
120 Returns:
121 0 in error case else 1
121 0 in error case else 1
122 """
122 """
123 raise NotImplementedError("Implement this method in child class")
123 raise NotImplementedError("Implement this method in child class")
124
124
125 def upload(self, fullfilename, remotefolder=None):
125 def upload(self, fullfilename, remotefolder=None):
126 """
126 """
127 upload method is used to upload a local file to remote directory. This method changes
127 upload method is used to upload a local file to remote directory. This method changes
128 working directory before sending a file.
128 working directory before sending a file.
129
129
130 Inputs:
130 Inputs:
131 fullfilename - full path name of local file to store in remote directory
131 fullfilename - full path name of local file to store in remote directory
132
132
133 remotefolder - remote directory
133 remotefolder - remote directory
134
134
135 Returns:
135 Returns:
136 0 in error case else 1
136 0 in error case else 1
137 """
137 """
138 print("[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder))
138 print("[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder))
139
139
140 if not self.status:
140 if not self.status:
141 return 0
141 return 0
142
142
143 if remotefolder == None:
143 if remotefolder == None:
144 remotefolder = self.remotefolder
144 remotefolder = self.remotefolder
145
145
146 if not self.cd(remotefolder):
146 if not self.cd(remotefolder):
147 return 0
147 return 0
148
148
149 if not self.sendFile(fullfilename):
149 if not self.sendFile(fullfilename):
150 print("[Remote Server] Error uploading file %s" %fullfilename)
150 print("[Remote Server] Error uploading file %s" %fullfilename)
151 return 0
151 return 0
152
152
153
153
154
154
155 return 1
155 return 1
156
156
157 def delete(self, filename):
157 def delete(self, filename):
158 """
158 """
159 Remove a file from remote server
159 Remove a file from remote server
160 """
160 """
161 pass
161 pass
162
162
163 def updateFileList(self, fileList):
163 def updateFileList(self, fileList):
164 """
164 """
165 Remove a file from remote server
165 Remove a file from remote server
166 """
166 """
167
167
168 if fileList == self.fileList:
168 if fileList == self.fileList:
169 return 0
169 return 0
170
170
171 self.mutex.acquire()
171 self.mutex.acquire()
172 # init = time.time()
172 # init = time.time()
173 #
173 #
174 # while(self.bussy):
174 # while(self.bussy):
175 # sleep(0.1)
175 # sleep(0.1)
176 # if time.time() - init > 2*self.period:
176 # if time.time() - init > 2*self.period:
177 # return 0
177 # return 0
178
178
179 self.fileList = fileList
179 self.fileList = fileList
180 self.mutex.release()
180 self.mutex.release()
181 return 1
181 return 1
182
182
183 def run(self):
183 def run(self):
184
184
185 if not self.status:
185 if not self.status:
186 print("Finishing FTP service")
186 print("Finishing FTP service")
187 return
187 return
188
188
189 if not self.cd(self.remotefolder):
189 if not self.cd(self.remotefolder):
190 raise ValueError("Could not access to the new remote directory: %s" %self.remotefolder)
190 raise ValueError("Could not access to the new remote directory: %s" %self.remotefolder)
191
191
192 while True:
192 while True:
193
193
194 for i in range(self.period):
194 for i in range(self.period):
195 if self.stopFlag:
195 if self.stopFlag:
196 break
196 break
197 sleep(1)
197 sleep(1)
198
198
199 if self.stopFlag:
199 if self.stopFlag:
200 break
200 break
201
201
202 # self.bussy = True
202 # self.bussy = True
203 self.mutex.acquire()
203 self.mutex.acquire()
204
204
205 print("[Remote Server] Opening %s" %self.__server)
205 print("[Remote Server] Opening %s" %self.__server)
206 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
206 if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
207 self.mutex.release()
207 self.mutex.release()
208 continue
208 continue
209
209
210 for thisFile in self.fileList:
210 for thisFile in self.fileList:
211 self.upload(thisFile, self.remotefolder)
211 self.upload(thisFile, self.remotefolder)
212
212
213 print("[Remote Server] Closing %s" %self.__server)
213 print("[Remote Server] Closing %s" %self.__server)
214 self.close()
214 self.close()
215
215
216 self.mutex.release()
216 self.mutex.release()
217 # self.bussy = False
217 # self.bussy = False
218
218
219 print("[Remote Server] Thread stopped successfully")
219 print("[Remote Server] Thread stopped successfully")
220
220
221 class FTPClient(Remote):
221 class FTPClient(Remote):
222
222
223 __ftpClientObj = None
223 __ftpClientObj = None
224
224
225 def __init__(self, server, username, password, remotefolder, period=60):
225 def __init__(self, server, username, password, remotefolder, period=60):
226 """
226 """
227 """
227 """
228 Remote.__init__(self, server, username, password, remotefolder, period)
228 Remote.__init__(self, server, username, password, remotefolder, period)
229
229
230 def open(self, server, username, password, remotefolder):
230 def open(self, server, username, password, remotefolder):
231
231
232 """
232 """
233 This method is used to set FTP parameters and establish a connection to remote server
233 This method is used to set FTP parameters and establish a connection to remote server
234
234
235 Inputs:
235 Inputs:
236 server - remote server IP Address
236 server - remote server IP Address
237
237
238 username - remote server Username
238 username - remote server Username
239
239
240 password - remote server password
240 password - remote server password
241
241
242 remotefolder - remote server current working directory
242 remotefolder - remote server current working directory
243
243
244 Return:
244 Return:
245 Boolean - Returns 1 if a connection has been established, 0 otherwise
245 Boolean - Returns 1 if a connection has been established, 0 otherwise
246
246
247 Affects:
247 Affects:
248 self.status - in case of error or fail connection this parameter is set to 0 else 1
248 self.status - in case of error or fail connection this parameter is set to 0 else 1
249
249
250 """
250 """
251
251
252 if server == None:
252 if server == None:
253 raise ValueError("FTP server should be defined")
253 raise ValueError("FTP server should be defined")
254
254
255 if username == None:
255 if username == None:
256 raise ValueError("FTP username should be defined")
256 raise ValueError("FTP username should be defined")
257
257
258 if password == None:
258 if password == None:
259 raise ValueError("FTP password should be defined")
259 raise ValueError("FTP password should be defined")
260
260
261 if remotefolder == None:
261 if remotefolder == None:
262 raise ValueError("FTP remote folder should be defined")
262 raise ValueError("FTP remote folder should be defined")
263
263
264 try:
264 try:
265 ftpClientObj = ftplib.FTP(server)
265 ftpClientObj = ftplib.FTP(server)
266 except ftplib.all_errors as e:
266 except ftplib.all_errors as e:
267 print("[FTP Server]: FTP server connection fail: %s" %server)
267 print("[FTP Server]: FTP server connection fail: %s" %server)
268 print("[FTP Server]:", e)
268 print("[FTP Server]:", e)
269 self.status = 0
269 self.status = 0
270 return 0
270 return 0
271
271
272 try:
272 try:
273 ftpClientObj.login(username, password)
273 ftpClientObj.login(username, password)
274 except ftplib.all_errors:
274 except ftplib.all_errors:
275 print("[FTP Server]: FTP username or password are incorrect")
275 print("[FTP Server]: FTP username or password are incorrect")
276 self.status = 0
276 self.status = 0
277 return 0
277 return 0
278
278
279 if remotefolder == None:
279 if remotefolder == None:
280 remotefolder = ftpClientObj.pwd()
280 remotefolder = ftpClientObj.pwd()
281 else:
281 else:
282 try:
282 try:
283 ftpClientObj.cwd(remotefolder)
283 ftpClientObj.cwd(remotefolder)
284 except ftplib.all_errors:
284 except ftplib.all_errors:
285 print("[FTP Server]: FTP remote folder is invalid: %s" %remotefolder)
285 print("[FTP Server]: FTP remote folder is invalid: %s" %remotefolder)
286 remotefolder = ftpClientObj.pwd()
286 remotefolder = ftpClientObj.pwd()
287
287
288 self.server = server
288 self.server = server
289 self.username = username
289 self.username = username
290 self.password = password
290 self.password = password
291 self.remotefolder = remotefolder
291 self.remotefolder = remotefolder
292 self.__ftpClientObj = ftpClientObj
292 self.__ftpClientObj = ftpClientObj
293 self.status = 1
293 self.status = 1
294
294
295 return 1
295 return 1
296
296
297 def close(self):
297 def close(self):
298 """
298 """
299 Close connection to remote server
299 Close connection to remote server
300 """
300 """
301 if not self.status:
301 if not self.status:
302 return 0
302 return 0
303
303
304 self.__ftpClientObj.close()
304 self.__ftpClientObj.close()
305
305
306 def mkdir(self, remotefolder):
306 def mkdir(self, remotefolder):
307 """
307 """
308 mkdir is used to make a new directory in remote server
308 mkdir is used to make a new directory in remote server
309
309
310 Input:
310 Input:
311 remotefolder - directory name
311 remotefolder - directory name
312
312
313 Return:
313 Return:
314 0 in error case else 1
314 0 in error case else 1
315 """
315 """
316 if not self.status:
316 if not self.status:
317 return 0
317 return 0
318
318
319 try:
319 try:
320 self.__ftpClientObj.mkd(dirname)
320 self.__ftpClientObj.mkd(dirname)
321 except ftplib.all_errors:
321 except ftplib.all_errors:
322 print("[FTP Server]: Error creating remote folder: %s" %remotefolder)
322 print("[FTP Server]: Error creating remote folder: %s" %remotefolder)
323 return 0
323 return 0
324
324
325 return 1
325 return 1
326
326
327 def cd(self, remotefolder):
327 def cd(self, remotefolder):
328 """
328 """
329 cd is used to change remote working directory on server
329 cd is used to change remote working directory on server
330
330
331 Input:
331 Input:
332 remotefolder - current working directory
332 remotefolder - current working directory
333
333
334 Affects:
334 Affects:
335 self.remotefolder
335 self.remotefolder
336
336
337 Return:
337 Return:
338 0 in case of error else 1
338 0 in case of error else 1
339 """
339 """
340 if not self.status:
340 if not self.status:
341 return 0
341 return 0
342
342
343 if remotefolder == self.remotefolder:
343 if remotefolder == self.remotefolder:
344 return 1
344 return 1
345
345
346 try:
346 try:
347 self.__ftpClientObj.cwd(remotefolder)
347 self.__ftpClientObj.cwd(remotefolder)
348 except ftplib.all_errors:
348 except ftplib.all_errors:
349 print('[FTP Server]: Error changing to %s' %remotefolder)
349 print('[FTP Server]: Error changing to %s' %remotefolder)
350 print('[FTP Server]: Trying to create remote folder')
350 print('[FTP Server]: Trying to create remote folder')
351
351
352 if not self.mkdir(remotefolder):
352 if not self.mkdir(remotefolder):
353 print('[FTP Server]: Remote folder could not be created')
353 print('[FTP Server]: Remote folder could not be created')
354 return 0
354 return 0
355
355
356 try:
356 try:
357 self.__ftpClientObj.cwd(remotefolder)
357 self.__ftpClientObj.cwd(remotefolder)
358 except ftplib.all_errors:
358 except ftplib.all_errors:
359 return 0
359 return 0
360
360
361 self.remotefolder = remotefolder
361 self.remotefolder = remotefolder
362
362
363 return 1
363 return 1
364
364
365 def sendFile(self, fullfilename):
365 def sendFile(self, fullfilename):
366
366
367 if not self.status:
367 if not self.status:
368 return 0
368 return 0
369
369
370 fp = open(fullfilename, 'rb')
370 fp = open(fullfilename, 'rb')
371
371
372 filename = os.path.basename(fullfilename)
372 filename = os.path.basename(fullfilename)
373
373
374 command = "STOR %s" %filename
374 command = "STOR %s" %filename
375
375
376 try:
376 try:
377 self.__ftpClientObj.storbinary(command, fp)
377 self.__ftpClientObj.storbinary(command, fp)
378 except ftplib.all_errors as e:
378 except ftplib.all_errors as e:
379 print("[FTP Server]:", e)
379 print("[FTP Server]:", e)
380 return 0
380 return 0
381
381
382 try:
382 try:
383 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
383 self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
384 except ftplib.all_errors as e:
384 except ftplib.all_errors as e:
385 print("[FTP Server]:", e)
385 print("[FTP Server]:", e)
386
386
387 fp.close()
387 fp.close()
388
388
389 return 1
389 return 1
390
390
391 class SSHClient(Remote):
391 class SSHClient(Remote):
392
392
393 __sshClientObj = None
393 __sshClientObj = None
394 __scpClientObj = None
394 __scpClientObj = None
395
395
396
396
397 def __init__(self, server, username, password, remotefolder, period=60,key_filename=None):
397 def __init__(self, server, username, password, remotefolder, period=60,key_filename=None):
398 """
398 """
399 """
399 """
400 Remote.__init__(self, server, username, password, remotefolder, period, key_filename)
400 Remote.__init__(self, server, username, password, remotefolder, period, key_filename)
401
401
402 def open(self, server, username, password, remotefolder, port=22, key_filename=None):
402 def open(self, server, username, password, remotefolder, port=22, key_filename=None):
403
403
404 """
404 """
405 This method is used to set SSH parameters and establish a connection to a remote server
405 This method is used to set SSH parameters and establish a connection to a remote server
406
406
407 Inputs:
407 Inputs:
408 server - remote server IP Address
408 server - remote server IP Address
409
409
410 username - remote server Username
410 username - remote server Username
411
411
412 password - remote server password
412 password - remote server password
413
413
414 remotefolder - remote server current working directory
414 remotefolder - remote server current working directory
415
415
416 key_filename - filename of the private key/optional
416 key_filename - filename of the private key/optional
417
417
418 Return: void
418 Return: void
419
419
420 Affects:
420 Affects:
421 self.status - in case of error or fail connection this parameter is set to 0 else 1
421 self.status - in case of error or fail connection this parameter is set to 0 else 1
422
422
423 """
423 """
424 #import socket
424 #import socket
425
425
426 if server == None:
426 if server == None:
427 raise ValueError("SSH server should be defined")
427 raise ValueError("SSH server should be defined")
428
428
429 if username == None:
429 if username == None:
430 raise ValueError("SSH username should be defined")
430 raise ValueError("SSH username should be defined")
431
431
432 if password == None:
432 if password == None:
433 raise ValueError("SSH password should be defined")
433 raise ValueError("SSH password should be defined")
434
434
435 if remotefolder == None:
435 if remotefolder == None:
436 raise ValueError("SSH remote folder should be defined")
436 raise ValueError("SSH remote folder should be defined")
437
437
438 self.__sshClientObj = paramiko.SSHClient()
438 self.__sshClientObj = paramiko.SSHClient()
439
439
440 self.__sshClientObj.load_system_host_keys()
440 self.__sshClientObj.load_system_host_keys()
441 self.__sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
441 self.__sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
442
442
443 self.status = 0
443 self.status = 0
444
444
445 try:
445 try:
446 if key_filename != None:
446 if key_filename != None:
447 self.__sshClientObj.connect(server, username=username, password=password, port=port, key_filename=key_filename)
447 self.__sshClientObj.connect(server, username=username, password=password, port=port, key_filename=key_filename)
448 else:
448 else:
449 self.__sshClientObj.connect(server, username=username, password=password, port=port)
449 self.__sshClientObj.connect(server, username=username, password=password, port=port)
450 except paramiko.AuthenticationException as e:
450 except paramiko.AuthenticationException as e:
451 # print "SSH username or password are incorrect: %s"
451 # print "SSH username or password are incorrect: %s"
452 print("[SSH Server]:", e)
452 print("[SSH Server]:", e)
453 return 0
453 return 0
454 # except SSHException as e:
454 # except SSHException as e:
455 # print("[SSH Server]:", e)
455 # print("[SSH Server]:", e)
456 # return 0
456 # return 0
457 # except socket.error:
457 # except socket.error:
458 # self.status = 0
458 # self.status = 0
459 # print("[SSH Server]:", e)
459 # print("[SSH Server]:", e)
460 # return 0
460 # return 0
461
461
462 self.status = 1
462 self.status = 1
463 #self.__scpClientObj = scp.SCPClient(self.__sshClientObj.get_transport(), socket_timeout=30)
463 #self.__scpClientObj = scp.SCPClient(self.__sshClientObj.get_transport(), socket_timeout=30)
464 self.__scpClientObj = self.__sshClientObj.open_sftp()
464 self.__scpClientObj = self.__sshClientObj.open_sftp()
465 if remotefolder == None:
465 if remotefolder == None:
466 remotefolder = self.pwd()
466 remotefolder = self.pwd()
467
467
468 self.server = server
468 self.server = server
469 self.username = username
469 self.username = username
470 self.password = password
470 self.password = password
471 # self.__sshClientObj = self.__sshClientObj
471 # self.__sshClientObj = self.__sshClientObj
472 # self.__scpClientObj = self.__scpClientObj
472 # self.__scpClientObj = self.__scpClientObj
473 self.status = 1
473 self.status = 1
474
474
475 if not self.cd(remotefolder):
475 if not self.cd(remotefolder):
476 raise ValueError("[SSH Server]: Could not access to remote folder: %s" %remotefolder)
476 raise ValueError("[SSH Server]: Could not access to remote folder: %s" %remotefolder)
477 return 0
477 return 0
478
478
479 self.remotefolder = remotefolder
479 self.remotefolder = remotefolder
480
480
481 return 1
481 return 1
482
482
483 def close(self):
483 def close(self):
484 """
484 """
485 Close connection to remote server
485 Close connection to remote server
486 """
486 """
487 if not self.status:
487 if not self.status:
488 return 0
488 return 0
489
489
490 self.__scpClientObj.close()
490 self.__scpClientObj.close()
491 self.__sshClientObj.close()
491 self.__sshClientObj.close()
492
492
493 def __execute(self, command):
493 def __execute(self, command):
494 """
494 """
495 __execute a command on remote server
495 __execute a command on remote server
496
496
497 Input:
497 Input:
498 command - Exmaple 'ls -l'
498 command - Exmaple 'ls -l'
499
499
500 Return:
500 Return:
501 0 in error case else 1
501 0 in error case else 1
502 """
502 """
503 if not self.status:
503 if not self.status:
504 return 0
504 return 0
505
505
506 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
506 stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
507
507
508 result = stderr.readlines()
508 result = stderr.readlines()
509 if len(result) > 1:
509 if len(result) > 1:
510 return 0
510 return 0
511
511
512 result = stdout.readlines()
512 result = stdout.readlines()
513 if len(result) > 1:
513 if len(result) > 1:
514 return result[0][:-1]
514 return result[0][:-1]
515
515
516 return 1
516 return 1
517
517
518 def mkdir(self, remotefolder):
518 def mkdir(self, remotefolder):
519 """
519 """
520 mkdir is used to make a new directory in remote server
520 mkdir is used to make a new directory in remote server
521
521
522 Input:
522 Input:
523 remotefolder - directory name
523 remotefolder - directory name
524
524
525 Return:
525 Return:
526 0 in error case else 1
526 0 in error case else 1
527 """
527 """
528
528
529 command = 'mkdir %s' %remotefolder
529 command = 'mkdir %s' %remotefolder
530
530
531 return self.__execute(command)
531 return self.__execute(command)
532
532
533 def pwd(self):
533 def pwd(self):
534
534
535 command = 'pwd'
535 command = 'pwd'
536
536
537 return self.__execute(command)
537 return self.__execute(command)
538
538
539 def cd(self, remotefolder):
539 def cd(self, remotefolder):
540 """
540 """
541 cd is used to change remote working directory on server
541 cd is used to change remote working directory on server
542
542
543 Input:
543 Input:
544 remotefolder - current working directory
544 remotefolder - current working directory
545
545
546 Affects:
546 Affects:
547 self.remotefolder
547 self.remotefolder
548
548
549 Return:
549 Return:
550 0 in case of error else 1
550 0 in case of error else 1
551 """
551 """
552 if not self.status:
552 if not self.status:
553 return 0
553 return 0
554
554
555 if remotefolder == self.remotefolder:
555 if remotefolder == self.remotefolder:
556 return 1
556 return 1
557
557
558 chk_command = "cd %s; pwd" %remotefolder
558 chk_command = "cd %s; pwd" %remotefolder
559 mkdir_command = "mkdir %s" %remotefolder
559 mkdir_command = "mkdir %s" %remotefolder
560
560
561 if not self.__execute(chk_command):
561 if not self.__execute(chk_command):
562 if not self.__execute(mkdir_command):
562 if not self.__execute(mkdir_command):
563 self.remotefolder = None
563 self.remotefolder = None
564 return 0
564 return 0
565
565
566 self.remotefolder = remotefolder
566 self.remotefolder = remotefolder
567
567
568 return 1
568 return 1
569
569
570 def sendFile(self, fullfilename):
570 def sendFile(self, fullfilename):
571
571
572 if not self.status:
572 if not self.status:
573 return 0
573 return 0
574
574
575 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
575 remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
576 print("remotefile",fullfilename, remotefile)
576 print("remotefile",fullfilename, remotefile)
577
577
578 try:
578 try:
579 self.__scpClientObj.put(fullfilename,remotefile)
579 self.__scpClientObj.put(fullfilename,remotefile)
580 except paramiko.SSHException as e:
580 except paramiko.SSHException as e:
581 print("[SSH Server]", str(e))
581 print("[SSH Server]", str(e))
582 print(fullfilename," to ",remotefile)
582 print(fullfilename," to ",remotefile)
583 return 0
583 return 0
584
584
585
585
586 #command = 'chmod 775 %s' %remotefile
586 #command = 'chmod 775 %s' %remotefile
587
587
588 return 1#self.__execute(command)
588 return 1#self.__execute(command)
589 #@MPDecorator
589 #@MPDecorator
590 class SendToServerProc(ProcessingUnit):
590 class SendToServerProc(ProcessingUnit):
591
591
592 sendByTrigger = False
592 sendByTrigger = False
593
593
594 def __init__(self, **kwargs):
594 def __init__(self, **kwargs):
595
595
596 ProcessingUnit.__init__(self)
596 ProcessingUnit.__init__(self)
597
597
598 self.isConfig = False
598 self.isConfig = False
599 self.clientObj = None
599 self.clientObj = None
600 self.dataOut = Parameters()
600 self.dataOut = Parameters()
601 self.dataOut.error=False
601 self.dataOut.error=False
602 self.dataOut.flagNoData=True
602 self.dataOut.flagNoData=True
603
603
604 def setup(self, server=None, username="", password="", remotefolder="", localfolder="",
604 def setup(self, server=None, username="", password="", remotefolder="", localfolder="",
605 ext='.png', period=60, protocol='ftp', sendByTrigger=False, key_filename=None):
605 ext='.png', period=60, protocol='ftp', sendByTrigger=False, key_filename=None):
606 self.server = server
606 self.server = server
607 self.username = username
607 self.username = username
608 self.password = password
608 self.password = password
609 self.remotefolder = remotefolder
609 self.remotefolder = remotefolder
610 self.clientObj = None
610 self.clientObj = None
611 self.localfolder = localfolder
611 self.localfolder = localfolder
612 self.ext = ext
612 self.ext = ext
613 self.sendByTrigger = sendByTrigger
613 self.sendByTrigger = sendByTrigger
614 self.period = period
614 self.period = period
615 self.key_filename = key_filename
615 self.key_filename = key_filename
616 if self.sendByTrigger:
616 if self.sendByTrigger:
617 self.period = 1000000000000 #para que no se ejecute por tiempo
617 self.period = 1000000000000 #para que no se ejecute por tiempo
618
618
619 if str.lower(protocol) == 'ftp':
619 if str.lower(protocol) == 'ftp':
620 self.clientObj = FTPClient(server, username, password, remotefolder, period)
620 self.clientObj = FTPClient(server, username, password, remotefolder, period)
621
621
622 if str.lower(protocol) == 'ssh':
622 if str.lower(protocol) == 'ssh':
623 self.clientObj = SSHClient(self.server, self.username, self.password,
623 self.clientObj = SSHClient(self.server, self.username, self.password,
624 self.remotefolder, period=600000,key_filename=self.key_filename)
624 self.remotefolder, period=600000,key_filename=self.key_filename)
625
625
626 if not self.clientObj:
626 if not self.clientObj:
627 raise ValueError("%s has been chosen as remote access protocol but it is not valid" %protocol)
627 raise ValueError("%s has been chosen as remote access protocol but it is not valid" %protocol)
628
628
629 #self.clientObj.start()
630 print("Send to Server setup complete")
629 print("Send to Server setup complete")
631
630
632
631
633 def findFiles(self):
632 def findFiles(self):
634
633
635 if not type(self.localfolder) == list:
634 if not type(self.localfolder) == list:
636 folderList = [self.localfolder]
635 folderList = [self.localfolder]
637 else:
636 else:
638 folderList = self.localfolder
637 folderList = self.localfolder
639
638
640 #Remove duplicate items
639 #Remove duplicate items
641 folderList = list(set(folderList))
640 folderList = list(set(folderList))
642
641
643 fullfilenameList = []
642 fullfilenameList = []
644
643
645 for thisFolder in folderList:
644 for thisFolder in folderList:
646
645
647 print("[Remote Server]: Searching files on %s" %thisFolder)
646 print("[Remote Server]: Searching files on %s" %thisFolder)
648
647
649 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
648 filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
650
649
651 if len(filenameList) < 1:
650 if len(filenameList) < 1:
652
651
653 continue
652 continue
654
653
655 for thisFile in filenameList:
654 for thisFile in filenameList:
656 fullfilename = os.path.join(thisFolder, thisFile)
655 fullfilename = os.path.join(thisFolder, thisFile)
657
656
658 if fullfilename in fullfilenameList:
657 if fullfilename in fullfilenameList:
659 continue
658 continue
660
659
661 #Only files modified in the last 30 minutes are considered
660 #Only files modified in the last 30 minutes are considered
662 if os.path.getmtime(fullfilename) < time.time() - 30*60:
661 if os.path.getmtime(fullfilename) < time.time() - 30*60:
663 continue
662 continue
664
663
665 fullfilenameList.append(fullfilename)
664 fullfilenameList.append(fullfilename)
666 fullfilenameList.sort()
665 fullfilenameList.sort()
667
666
668 return fullfilenameList
667 return fullfilenameList
669
668
670 def run(self, **kwargs):
669 def run(self, **kwargs):
671
670
672 if not self.isConfig:
671 if not self.isConfig:
673 self.init = time.time()
672 self.init = time.time()
674 self.setup(**kwargs)
673 self.setup(**kwargs)
675 self.isConfig = True
674 self.isConfig = True
676
675
677 if not self.clientObj.is_alive():
676 if not self.clientObj.is_alive():
678 print("[Remote Server]: Restarting connection ")
677 print("[Remote Server]: Restarting connection ")
679 self.setup( **kwargs)
678 self.setup( **kwargs)
680
679
681 if ((time.time() - self.init) >= self.period and not self.sendByTrigger) or (self.sendByTrigger and not self.dataIn.flagNoData):
680 if ((time.time() - self.init) >= self.period and not self.sendByTrigger) or (self.sendByTrigger and not self.dataIn.flagNoData):
682 fullfilenameList = self.findFiles()
681 fullfilenameList = self.findFiles()
683 if self.sendByTrigger:
682 if self.sendByTrigger:
684 if self.clientObj.upload(fullfilenameList[-1]): #last file to send
683 if self.clientObj.upload(fullfilenameList[-1]): #last file to send
685 print("[Remote Server] upload finished successfully")
684 print("[Remote Server] upload finished successfully")
686 else:
685 else:
687 for file in fullfilenameList:
686 for file in fullfilenameList:
688 self.clientObj.upload(file)
687 self.clientObj.upload(file)
689
688
690 # if self.clientObj.updateFileList(fullfilenameList):
689 # if self.clientObj.updateFileList(fullfilenameList):
691 # print("[Remote Server]: Sending the next files ", str(fullfilenameList))
690 # print("[Remote Server]: Sending the next files ", str(fullfilenameList))
692
691
693 self.init = time.time()
692 self.init = time.time()
694
693
695 def close(self):
694 def close(self):
696 print("[Remote Server] Stopping thread")
695 print("[Remote Server] Stopping thread")
697 self.clientObj.stop()
696 self.clientObj.stop()
698
697
699 class SendByRSYNCProc(ProcessingUnit):
698 class SendByRSYNCProc(ProcessingUnit):
700
699
701 sendByTrigger = False
700 sendByTrigger = False
702
701
703 def __init__(self, **kwargs):
702 def __init__(self, **kwargs):
704
703
705 ProcessingUnit.__init__(self)
704 ProcessingUnit.__init__(self)
706
705
707 self.isConfig = False
706 self.isConfig = False
708 self.dataOut = Parameters()
707 self.dataOut = Parameters()
709 self.dataOut.error=False
708 self.dataOut.error=False
710 self.dataOut.flagNoData=True
709 self.dataOut.flagNoData=True
711
710
712 def setup(self, server="", username="", remotefolder="", localfolder="",sendByTrigger=True,
711 def setup(self, server="", username="", remotefolder="", localfolder="",sendByTrigger=True,
713 period=60, key_filename=None, port=22 ,param1="", param2=""):
712 period=60, key_filename=None, port=22 ,param1="", param2=""):
714 self.server = server
713 self.server = server
715 self.username = username
714 self.username = username
716 self.remotefolder = remotefolder
715 self.remotefolder = remotefolder
717 self.localfolder = localfolder
716 self.localfolder = localfolder
718 self.period = period
717 self.period = period
719 self.key_filename = key_filename
718 self.key_filename = key_filename
720 if type(param1)==str:
719 if type(param1)==str:
721 self.param1 = list(param1.split(","))
720 self.param1 = list(param1.split(","))
722 else:
721 else:
723 self.param1 = param1
722 self.param1 = param1
724 if type(param2)==str:
723 if type(param2)==str:
725 self.param2 = list(param2.split(","))
724 self.param2 = list(param2.split(","))
726 else:
725 else:
727 self.param2 = param2
726 self.param2 = param2
728 self.port = port
727 self.port = port
729 self.sendByTrigger = sendByTrigger
728 self.sendByTrigger = sendByTrigger
730 if self.sendByTrigger:
729 if self.sendByTrigger:
731 self.period = 1000000000000 #para que no se ejecute por tiempo
730 self.period = 1000000000000 #para que no se ejecute por tiempo
732 self.command ="rsync "
731 self.command ="rsync "
733
732
734 def syncFolders(self):
733 def syncFolders(self):
735 self.command ="rsync "
734 self.command ="rsync "
736 for p1 in self.param1:
735 for p1 in self.param1:
737 self.command += " -"+str(p1)
736 self.command += " -"+str(p1)
738 for p2 in self.param2:
737 for p2 in self.param2:
739 self.command += " --"+str(p2)
738 self.command += " --"+str(p2)
740 if self.key_filename != None:
739 if self.key_filename != None:
741 self.command += """ "ssh -i {} -p {}" """.format(self.key_filename, self.port)
740 self.command += """ "ssh -i {} -p {}" """.format(self.key_filename, self.port)
742 self.command += " {} ".format(self.localfolder)
741 self.command += " {} ".format(self.localfolder)
743 self.command += " {}@{}:{}".format(self.username,self.server,self.remotefolder)
742 self.command += " {}@{}:{}".format(self.username,self.server,self.remotefolder)
744 print("CMD: ",self.command)
743 print("CMD: ",self.command)
745 #os.system(self.command)
744 #os.system(self.command)
746 return
745 return
747
746
748 def run(self, **kwargs):
747 def run(self, **kwargs):
749
748
750 if not self.isConfig:
749 if not self.isConfig:
751 self.init = time.time()
750 self.init = time.time()
752 self.setup(**kwargs)
751 self.setup(**kwargs)
753 self.isConfig = True
752 self.isConfig = True
754
753
755 if self.sendByTrigger and not self.dataIn.flagNoData:
754 if self.sendByTrigger and not self.dataIn.flagNoData:
756 self.syncFolders()
755 self.syncFolders()
757 else:
756 else:
758 if (time.time() - self.init) >= self.period:
757 if (time.time() - self.init) >= self.period:
759 self.syncFolders()
758 self.syncFolders()
760 self.init = time.time()
759 self.init = time.time()
761
760
762 return
761 return
763
762
764
763
765
764
766 class FTP(object):
765 class FTP(object):
767 """
766 """
768 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
767 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
769
768
770 Non-standard Python modules used: None
769 Non-standard Python modules used: None
771
770
772 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
771 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe Oct. 26, 2010
773 """
772 """
774
773
775 def __init__(self,server = None, username=None, password=None, remotefolder=None):
774 def __init__(self,server = None, username=None, password=None, remotefolder=None):
776 """
775 """
777 This method is used to setting parameters for FTP and establishing connection to remote server
776 This method is used to setting parameters for FTP and establishing connection to remote server
778
777
779 Inputs:
778 Inputs:
780 server - remote server IP Address
779 server - remote server IP Address
781
780
782 username - remote server Username
781 username - remote server Username
783
782
784 password - remote server password
783 password - remote server password
785
784
786 remotefolder - remote server current working directory
785 remotefolder - remote server current working directory
787
786
788 Return: void
787 Return: void
789
788
790 Affects:
789 Affects:
791 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
790 self.status - in Error Case or Connection Failed this parameter is set to 1 else 0
792
791
793 self.folderList - sub-folder list of remote folder
792 self.folderList - sub-folder list of remote folder
794
793
795 self.fileList - file list of remote folder
794 self.fileList - file list of remote folder
796
795
797
796
798 """
797 """
799
798
800 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
799 if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
801 server, username, password, remotefolder = self.parmsByDefault()
800 server, username, password, remotefolder = self.parmsByDefault()
802
801
803 self.server = server
802 self.server = server
804 self.username = username
803 self.username = username
805 self.password = password
804 self.password = password
806 self.remotefolder = remotefolder
805 self.remotefolder = remotefolder
807 self.file = None
806 self.file = None
808 self.ftp = None
807 self.ftp = None
809 self.status = 0
808 self.status = 0
810
809
811 try:
810 try:
812 self.ftp = ftplib.FTP(self.server)
811 self.ftp = ftplib.FTP(self.server)
813 self.ftp.login(self.username,self.password)
812 self.ftp.login(self.username,self.password)
814 self.ftp.cwd(self.remotefolder)
813 self.ftp.cwd(self.remotefolder)
815 # print 'Connect to FTP Server: Successfully'
814 # print 'Connect to FTP Server: Successfully'
816
815
817 except ftplib.all_errors:
816 except ftplib.all_errors:
818 print('Error FTP Service')
817 print('Error FTP Service')
819 self.status = 1
818 self.status = 1
820 return
819 return
821
820
822
821
823
822
824 self.dirList = []
823 self.dirList = []
825
824
826 try:
825 try:
827 self.dirList = self.ftp.nlst()
826 self.dirList = self.ftp.nlst()
828
827
829 except ftplib.error_perm as resp:
828 except ftplib.error_perm as resp:
830 if str(resp) == "550 No files found":
829 if str(resp) == "550 No files found":
831 print("no files in this directory")
830 print("no files in this directory")
832 self.status = 1
831 self.status = 1
833 return
832 return
834
833
835 except ftplib.all_errors:
834 except ftplib.all_errors:
836 print('Error Displaying Dir-Files')
835 print('Error Displaying Dir-Files')
837 self.status = 1
836 self.status = 1
838 return
837 return
839
838
840 self.fileList = []
839 self.fileList = []
841 self.folderList = []
840 self.folderList = []
842 #only for test
841 #only for test
843 for f in self.dirList:
842 for f in self.dirList:
844 name, ext = os.path.splitext(f)
843 name, ext = os.path.splitext(f)
845 if ext != '':
844 if ext != '':
846 self.fileList.append(f)
845 self.fileList.append(f)
847 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
846 # print 'filename: %s - size: %d'%(f,self.ftp.size(f))
848
847
849 def parmsByDefault(self):
848 def parmsByDefault(self):
850 server = 'jro-app.igp.gob.pe'
849 server = 'jro-app.igp.gob.pe'
851 username = 'wmaster'
850 username = 'wmaster'
852 password = 'mst2010vhf'
851 password = 'mst2010vhf'
853 remotefolder = '/home/wmaster/graficos'
852 remotefolder = '/home/wmaster/graficos'
854
853
855 return server, username, password, remotefolder
854 return server, username, password, remotefolder
856
855
857
856
858 def mkd(self,dirname):
857 def mkd(self,dirname):
859 """
858 """
860 mkd is used to make directory in remote server
859 mkd is used to make directory in remote server
861
860
862 Input:
861 Input:
863 dirname - directory name
862 dirname - directory name
864
863
865 Return:
864 Return:
866 1 in error case else 0
865 1 in error case else 0
867 """
866 """
868 try:
867 try:
869 self.ftp.mkd(dirname)
868 self.ftp.mkd(dirname)
870 except:
869 except:
871 print('Error creating remote folder:%s'%dirname)
870 print('Error creating remote folder:%s'%dirname)
872 return 1
871 return 1
873
872
874 return 0
873 return 0
875
874
876
875
877 def delete(self,filename):
876 def delete(self,filename):
878 """
877 """
879 delete is used to delete file in current working directory of remote server
878 delete is used to delete file in current working directory of remote server
880
879
881 Input:
880 Input:
882 filename - filename to delete in remote folder
881 filename - filename to delete in remote folder
883
882
884 Return:
883 Return:
885 1 in error case else 0
884 1 in error case else 0
886 """
885 """
887
886
888 try:
887 try:
889 self.ftp.delete(filename)
888 self.ftp.delete(filename)
890 except:
889 except:
891 print('Error deleting remote file:%s'%filename)
890 print('Error deleting remote file:%s'%filename)
892 return 1
891 return 1
893
892
894 return 0
893 return 0
895
894
896 def download(self,filename,localfolder):
895 def download(self,filename,localfolder):
897 """
896 """
898 download is used to downloading file from remote folder into local folder
897 download is used to downloading file from remote folder into local folder
899
898
900 Inputs:
899 Inputs:
901 filename - filename to donwload
900 filename - filename to donwload
902
901
903 localfolder - directory local to store filename
902 localfolder - directory local to store filename
904
903
905 Returns:
904 Returns:
906 self.status - 1 in error case else 0
905 self.status - 1 in error case else 0
907 """
906 """
908
907
909 self.status = 0
908 self.status = 0
910
909
911
910
912 if not(filename in self.fileList):
911 if not(filename in self.fileList):
913 print('filename:%s not exists'%filename)
912 print('filename:%s not exists'%filename)
914 self.status = 1
913 self.status = 1
915 return self.status
914 return self.status
916
915
917 newfilename = os.path.join(localfolder,filename)
916 newfilename = os.path.join(localfolder,filename)
918
917
919 self.file = open(newfilename, 'wb')
918 self.file = open(newfilename, 'wb')
920
919
921 try:
920 try:
922 print('Download: ' + filename)
921 print('Download: ' + filename)
923 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
922 self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
924 print('Download Complete')
923 print('Download Complete')
925 except ftplib.all_errors:
924 except ftplib.all_errors:
926 print('Error Downloading ' + filename)
925 print('Error Downloading ' + filename)
927 self.status = 1
926 self.status = 1
928 return self.status
927 return self.status
929
928
930 self.file.close()
929 self.file.close()
931
930
932 return self.status
931 return self.status
933
932
934
933
935 def __handleDownload(self,block):
934 def __handleDownload(self,block):
936 """
935 """
937 __handleDownload is used to handle writing file
936 __handleDownload is used to handle writing file
938 """
937 """
939 self.file.write(block)
938 self.file.write(block)
940
939
941
940
942 def upload(self,filename,remotefolder=None):
941 def upload(self,filename,remotefolder=None):
943 """
942 """
944 upload is used to uploading local file to remote directory
943 upload is used to uploading local file to remote directory
945
944
946 Inputs:
945 Inputs:
947 filename - full path name of local file to store in remote directory
946 filename - full path name of local file to store in remote directory
948
947
949 remotefolder - remote directory
948 remotefolder - remote directory
950
949
951 Returns:
950 Returns:
952 self.status - 1 in error case else 0
951 self.status - 1 in error case else 0
953 """
952 """
954
953
955 if remotefolder == None:
954 if remotefolder == None:
956 remotefolder = self.remotefolder
955 remotefolder = self.remotefolder
957
956
958 self.status = 0
957 self.status = 0
959
958
960 try:
959 try:
961 self.ftp.cwd(remotefolder)
960 self.ftp.cwd(remotefolder)
962
961
963 self.file = open(filename, 'rb')
962 self.file = open(filename, 'rb')
964
963
965 (head, tail) = os.path.split(filename)
964 (head, tail) = os.path.split(filename)
966
965
967 command = "STOR " + tail
966 command = "STOR " + tail
968
967
969 print('Uploading: ' + tail)
968 print('Uploading: ' + tail)
970 self.ftp.storbinary(command, self.file)
969 self.ftp.storbinary(command, self.file)
971 print('Upload Completed')
970 print('Upload Completed')
972
971
973 except ftplib.all_errors:
972 except ftplib.all_errors:
974 print('Error Uploading ' + tail)
973 print('Error Uploading ' + tail)
975 self.status = 1
974 self.status = 1
976 return self.status
975 return self.status
977
976
978 self.file.close()
977 self.file.close()
979
978
980 #back to initial directory in __init__()
979 #back to initial directory in __init__()
981 self.ftp.cwd(self.remotefolder)
980 self.ftp.cwd(self.remotefolder)
982
981
983 return self.status
982 return self.status
984
983
985
984
986 def dir(self,remotefolder):
985 def dir(self,remotefolder):
987 """
986 """
988 dir is used to change working directory of remote server and get folder and file list
987 dir is used to change working directory of remote server and get folder and file list
989
988
990 Input:
989 Input:
991 remotefolder - current working directory
990 remotefolder - current working directory
992
991
993 Affects:
992 Affects:
994 self.fileList - file list of working directory
993 self.fileList - file list of working directory
995
994
996 Return:
995 Return:
997 infoList - list with filenames and size of file in bytes
996 infoList - list with filenames and size of file in bytes
998
997
999 self.folderList - folder list
998 self.folderList - folder list
1000 """
999 """
1001
1000
1002 self.remotefolder = remotefolder
1001 self.remotefolder = remotefolder
1003 print('Change to ' + self.remotefolder)
1002 print('Change to ' + self.remotefolder)
1004 try:
1003 try:
1005 self.ftp.cwd(remotefolder)
1004 self.ftp.cwd(remotefolder)
1006 except ftplib.all_errors:
1005 except ftplib.all_errors:
1007 print('Error Change to ' + self.remotefolder)
1006 print('Error Change to ' + self.remotefolder)
1008 infoList = None
1007 infoList = None
1009 self.folderList = None
1008 self.folderList = None
1010 return infoList,self.folderList
1009 return infoList,self.folderList
1011
1010
1012 self.dirList = []
1011 self.dirList = []
1013
1012
1014 try:
1013 try:
1015 self.dirList = self.ftp.nlst()
1014 self.dirList = self.ftp.nlst()
1016
1015
1017 except ftplib.error_perm as resp:
1016 except ftplib.error_perm as resp:
1018 if str(resp) == "550 No files found":
1017 if str(resp) == "550 No files found":
1019 print("no files in this directory")
1018 print("no files in this directory")
1020 infoList = None
1019 infoList = None
1021 self.folderList = None
1020 self.folderList = None
1022 return infoList,self.folderList
1021 return infoList,self.folderList
1023 except ftplib.all_errors:
1022 except ftplib.all_errors:
1024 print('Error Displaying Dir-Files')
1023 print('Error Displaying Dir-Files')
1025 infoList = None
1024 infoList = None
1026 self.folderList = None
1025 self.folderList = None
1027 return infoList,self.folderList
1026 return infoList,self.folderList
1028
1027
1029 infoList = []
1028 infoList = []
1030 self.fileList = []
1029 self.fileList = []
1031 self.folderList = []
1030 self.folderList = []
1032 for f in self.dirList:
1031 for f in self.dirList:
1033 name,ext = os.path.splitext(f)
1032 name,ext = os.path.splitext(f)
1034 if ext != '':
1033 if ext != '':
1035 self.fileList.append(f)
1034 self.fileList.append(f)
1036 value = (f,self.ftp.size(f))
1035 value = (f,self.ftp.size(f))
1037 infoList.append(value)
1036 infoList.append(value)
1038
1037
1039 if ext == '':
1038 if ext == '':
1040 self.folderList.append(f)
1039 self.folderList.append(f)
1041
1040
1042 return infoList,self.folderList
1041 return infoList,self.folderList
1043
1042
1044
1043
1045 def close(self):
1044 def close(self):
1046 """
1045 """
1047 close is used to close and end FTP connection
1046 close is used to close and end FTP connection
1048
1047
1049 Inputs: None
1048 Inputs: None
1050
1049
1051 Return: void
1050 Return: void
1052
1051
1053 """
1052 """
1054 self.ftp.close()
1053 self.ftp.close()
1055 @MPDecorator
1054 @MPDecorator
1056 class SendByFTP(Operation):
1055 class SendByFTP(Operation):
1057
1056
1058 def __init__(self, **kwargs):
1057 def __init__(self, **kwargs):
1059 Operation.__init__(self, **kwargs)
1058 Operation.__init__(self, **kwargs)
1060 self.status = 1
1059 self.status = 1
1061 self.counter = 0
1060 self.counter = 0
1062
1061
1063 def error_print(self, ValueError):
1062 def error_print(self, ValueError):
1064
1063
1065 print(ValueError, 'Error FTP')
1064 print(ValueError, 'Error FTP')
1066 print("don't worry the program is running...")
1065 print("don't worry the program is running...")
1067
1066
1068 def worker_ftp(self, server, username, password, remotefolder, filenameList):
1067 def worker_ftp(self, server, username, password, remotefolder, filenameList):
1069
1068
1070 self.ftpClientObj = FTP(server, username, password, remotefolder)
1069 self.ftpClientObj = FTP(server, username, password, remotefolder)
1071 for filename in filenameList:
1070 for filename in filenameList:
1072 self.ftpClientObj.upload(filename)
1071 self.ftpClientObj.upload(filename)
1073 self.ftpClientObj.close()
1072 self.ftpClientObj.close()
1074
1073
1075 def ftp_thread(self, server, username, password, remotefolder):
1074 def ftp_thread(self, server, username, password, remotefolder):
1076 if not(self.status):
1075 if not(self.status):
1077 return
1076 return
1078
1077
1079 import multiprocessing
1078 import multiprocessing
1080
1079
1081 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
1080 p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
1082 p.start()
1081 p.start()
1083
1082
1084 p.join(3)
1083 p.join(3)
1085
1084
1086 if p.is_alive():
1085 if p.is_alive():
1087 p.terminate()
1086 p.terminate()
1088 p.join()
1087 p.join()
1089 print('killing ftp process...')
1088 print('killing ftp process...')
1090 self.status = 0
1089 self.status = 0
1091 return
1090 return
1092
1091
1093 self.status = 1
1092 self.status = 1
1094 return
1093 return
1095
1094
1096 def filterByExt(self, ext, localfolder):
1095 def filterByExt(self, ext, localfolder):
1097 fnameList = glob.glob1(localfolder,ext)
1096 fnameList = glob.glob1(localfolder,ext)
1098 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
1097 self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
1099
1098
1100 if len(self.filenameList) == 0:
1099 if len(self.filenameList) == 0:
1101 self.status = 0
1100 self.status = 0
1102
1101
1103 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
1102 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
1104
1103
1105 self.counter += 1
1104 self.counter += 1
1106 if self.counter >= period:
1105 if self.counter >= period:
1107 self.filterByExt(ext, localfolder)
1106 self.filterByExt(ext, localfolder)
1108
1107
1109 self.ftp_thread(server, username, password, remotefolder)
1108 self.ftp_thread(server, username, password, remotefolder)
1110
1109
1111 self.counter = 0
1110 self.counter = 0
1112
1111
1113 self.status = 1
1112 self.status = 1
General Comments 0
You need to be logged in to leave comments. Login now