From e93e8802031b15bc3b527343a7b874b799527de1 2015-06-04 22:45:24 From: Miguel Valdez Date: 2015-06-04 22:45:24 Subject: [PATCH] A new SendToServer Unit has been created to upload files to a remote server. A testSendGraphs2server.py has been added to test this new feature. Test files have been updated --- diff --git a/schainpy/controller.py b/schainpy/controller.py index 242e389..abdae15 100644 --- a/schainpy/controller.py +++ b/schainpy/controller.py @@ -378,15 +378,6 @@ class ProcUnitConf(): self.name = upElement.get('name') self.datatype = upElement.get('datatype') self.inputId = upElement.get('inputId') - - #Compatible with old signal chain version - if self.ELEMENTNAME == ReadUnitConf().getElementName(): - if 'Reader' not in self.name: - self.name += 'Reader' - - if self.ELEMENTNAME == ProcUnitConf().getElementName(): - if 'Proc' not in self.name: - self.name += 'Proc' self.opConfObjList = [] @@ -445,7 +436,13 @@ class ProcUnitConf(): finalSts = finalSts or sts return finalSts - + + def close(self): + + self.procUnitObj.close() + + return + class ReadUnitConf(ProcUnitConf): path = None @@ -559,7 +556,7 @@ class Project(): return readUnitConfObj - def addProcUnit(self, inputId, datatype=None, name=None): + def addProcUnit(self, inputId=0, datatype=None, name=None): #Compatible with old signal chain version if datatype==None and name==None: @@ -707,7 +704,21 @@ class Project(): if not(finalSts): print "Every process unit have finished" break - + + #Closing every process + for procKey in keyList: + procUnitConfObj = self.procUnitConfObjDict[procKey] + procUnitConfObj.close() + + def start(self, filename): + + self.writeXml(filename) + self.readXml(filename) + + self.createObjects() + self.connectObjects() + self.run() + if __name__ == '__main__': desc = "Segundo Test" diff --git a/schainpy/model/graphics/figure.py b/schainpy/model/graphics/figure.py index c50b3fc..943d61d 100644 --- a/schainpy/model/graphics/figure.py +++ b/schainpy/model/graphics/figure.py @@ -39,6 +39,10 @@ class Figure: xmin = None xmax = None + counter_imagwr = 0 + + figfile = None + def __init__(self): raise ValueError, "This method is not implemented" @@ -175,6 +179,7 @@ class Figure: show=show) self.axesObjList = [] + self.counter_imagwr = 0 def setDriver(self, driver=mpldriver): @@ -224,8 +229,45 @@ class Figure: self.__driver.saveFigure(self.fig, filename, *args) - - + def save(self, figpath, figfile=None, save=True, ftp=False, wr_period=1, thisDatetime=None, update_figfile=True): + + if not save: + return + + if figfile == None: + + if not thisDatetime: + raise ValueError, "Saving figure: figfile or thisDatetime should be defined" + return + + str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") + figfile = self.getFilename(name = str_datetime) + + if self.figfile == None: + self.figfile = figfile + + if update_figfile: + self.figfile = figfile + + self.counter_imagwr += 1 + + if self.counter_imagwr=wr_period): - # store png plot to local folder - self.saveFigure(figpath, figfile) - # store png plot to FTP server according to RT-Web format - if ftp: - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) - self.counter_imagwr = 0 + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) diff --git a/schainpy/model/graphics/jroplot_heispectra.py b/schainpy/model/graphics/jroplot_heispectra.py index 82b52cf..5538d59 100644 --- a/schainpy/model/graphics/jroplot_heispectra.py +++ b/schainpy/model/graphics/jroplot_heispectra.py @@ -156,21 +156,12 @@ class SpectraHeisScope(Figure): self.draw() - if save: - - if figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, figfile) - # store png plot to FTP server according to RT-Web format - #name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - #ftp_filename = os.path.join(figpath, name) - #self.saveFigure(figpath, ftp_filename) - self.counter_imagwr = 0 + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) class RTIfromSpectraHeis(Figure): @@ -310,19 +301,12 @@ class RTIfromSpectraHeis(Figure): del self.xdata del self.ydata self.__isConfig = False - - if save: + self.figfile = None - if self.figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - self.figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, self.figfile) - # store png plot to FTP server according to RT-Web format - #name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - #ftp_filename = os.path.join(figpath, name) - #self.saveFigure(figpath, ftp_filename) - self.counter_imagwr = 0 + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime, + update_figfile=False) diff --git a/schainpy/model/graphics/jroplot_parameters.py b/schainpy/model/graphics/jroplot_parameters.py index 0545fdc..43dccaa 100644 --- a/schainpy/model/graphics/jroplot_parameters.py +++ b/schainpy/model/graphics/jroplot_parameters.py @@ -3,6 +3,7 @@ import datetime import numpy from figure import Figure, isRealtime +from plotting_codes import * class MomentsPlot(Figure): @@ -24,7 +25,8 @@ class MomentsPlot(Figure): self.HEIGHTPROF = 0 self.counter_imagwr = 0 - self.PLOT_CODE = 1 + self.PLOT_CODE = MOMENTS_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -179,23 +181,13 @@ class MomentsPlot(Figure): axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2) self.draw() - - if save: - - if figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - figfile = self.getFilename(name = str_datetime) - self.counter_imagwr += 1 - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, figfile) - self.counter_imagwr = 0 - # store png plot to FTP server according to RT-Web format - if ftp: - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) @@ -221,7 +213,8 @@ class SkyMapPlot(Figure): self.HEIGHTPROF = 0 self.counter_imagwr = 0 - self.PLOT_CODE = 1 + self.PLOT_CODE = SKYMAP_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -324,35 +317,13 @@ class SkyMapPlot(Figure): ticksize=9, cblabel='') self.draw() - - if save: - self.counter_imagwr += 1 - if (self.counter_imagwr==wr_period): - - if figfile == None: - figfile = self.getFilename(name = self.name) - - self.saveFigure(figpath, figfile) - self.counter_imagwr = 0 - - if ftp: - #provisionalmente envia archivos en el formato de la web en tiempo real - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - path = '%s%03d' %(self.PREFIX, self.id) - ftp_file = os.path.join(path,'ftp','%s.png'%name) - self.saveFigure(figpath, ftp_file) - ftp_filename = os.path.join(figpath,ftp_file) - - - try: - self.sendByFTP(ftp_filename, server, folder, username, password) - except: - self.counter_imagwr = 0 - raise ValueError, 'Error FTP' - - - + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) class WindProfilerPlot(Figure): @@ -375,7 +346,8 @@ class WindProfilerPlot(Figure): self.HEIGHTPROF = 0 self.counter_imagwr = 0 - self.PLOT_CODE = 0 + self.PLOT_CODE = WIND_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -559,32 +531,19 @@ class WindProfilerPlot(Figure): self.draw() - if save: - - if self.figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - self.figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 - - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, self.figfile) - self.counter_imagwr = 0 - - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) - - - if x[1] >= self.axesList[0].xmax: self.counter_imagwr = wr_period self.__isConfig = False self.figfile = None + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime, + update_figfile=False) + class ParametersPlot(Figure): @@ -607,7 +566,8 @@ class ParametersPlot(Figure): self.HEIGHTPROF = 0 self.counter_imagwr = 0 - self.PLOT_CODE = 0 + self.PLOT_CODE = PARMS_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -786,31 +746,19 @@ class ParametersPlot(Figure): self.draw() - if save: - - if self.figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - self.figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 - - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, self.figfile) - self.counter_imagwr = 0 - - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) - if x[1] >= self.axesList[0].xmax: self.counter_imagwr = wr_period self.__isConfig = False self.figfile = None - + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime, + update_figfile=False) + class SpectralFittingPlot(Figure): __isConfig = None @@ -828,6 +776,8 @@ class SpectralFittingPlot(Figure): self.__isConfig = False self.__nsubplots = 1 + self.PLOT_CODE = SPECFIT_CODE + self.WIDTH = 450 self.HEIGHT = 250 self.WIDTHPROF = 0 @@ -977,13 +927,13 @@ class SpectralFittingPlot(Figure): linestyle='solid', grid='both') self.draw() - - if save: - date = thisDatetime.strftime("%Y%m%d_%H%M%S") - if figfile == None: - figfile = self.getFilename(name = date) - - self.saveFigure(figpath, figfile) + + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) class EWDriftsPlot(Figure): @@ -1007,7 +957,8 @@ class EWDriftsPlot(Figure): self.HEIGHTPROF = 0 self.counter_imagwr = 0 - self.PLOT_CODE = 0 + self.PLOT_CODE = EWDRIFT_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -1183,26 +1134,15 @@ class EWDriftsPlot(Figure): self.draw() - if save: - - if self.figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - self.figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 - - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, self.figfile) - self.counter_imagwr = 0 - - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) - if x[1] >= self.axesList[0].xmax: self.counter_imagwr = wr_period self.__isConfig = False - self.figfile = None \ No newline at end of file + self.figfile = None + + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime, + update_figfile=False) \ No newline at end of file diff --git a/schainpy/model/graphics/jroplot_spectra.py b/schainpy/model/graphics/jroplot_spectra.py index 26b09ac..75e6997 100644 --- a/schainpy/model/graphics/jroplot_spectra.py +++ b/schainpy/model/graphics/jroplot_spectra.py @@ -8,6 +8,7 @@ import datetime import numpy from figure import Figure, isRealtime +from plotting_codes import * class SpectraPlot(Figure): @@ -29,7 +30,8 @@ class SpectraPlot(Figure): self.HEIGHTPROF = 0 self.counter_imagwr = 0 - self.PLOT_CODE = 1 + self.PLOT_CODE = SPEC_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -188,31 +190,20 @@ class SpectraPlot(Figure): axes.addpline(noiseline, y, idline=1, color="black", linestyle="dashed", lw=2) self.draw() - - if save: - - if figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - figfile = self.getFilename(name = str_datetime) - name = str_datetime - if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)): - name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith) - figfile = self.getFilename(name) - - self.counter_imagwr += 1 - - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, figfile) - self.counter_imagwr = 0 + + if figfile == None: + str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") + name = str_datetime + if ((dataOut.azimuth!=None) and (dataOut.zenith!=None)): + name = name + '_az' + '_%2.2f'%(dataOut.azimuth) + '_zn' + '_%2.2f'%(dataOut.zenith) + figfile = self.getFilename(name) - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) - - + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) class CrossSpectraPlot(Figure): @@ -235,7 +226,7 @@ class CrossSpectraPlot(Figure): self.WIDTHPROF = 0 self.HEIGHTPROF = 0 - self.PLOT_CODE = 1 + self.PLOT_CODE = CROSS_CODE self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -397,24 +388,12 @@ class CrossSpectraPlot(Figure): self.draw() - if save != '': - - if figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 - - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, figfile) - self.counter_imagwr = 0 - - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) class RTIPlot(Figure): @@ -438,7 +417,8 @@ class RTIPlot(Figure): self.HEIGHTPROF = 0 self.counter_imagwr = 0 - self.PLOT_CODE = 0 + self.PLOT_CODE = RTI_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -603,30 +583,19 @@ class RTIPlot(Figure): grid='x') self.draw() - - if save: - if self.figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - self.figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 - - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, self.figfile) - self.counter_imagwr = 0 - - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) - if x[1] >= self.axesList[0].xmax: self.counter_imagwr = wr_period self.__isConfig = False self.figfile = None + + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime, + update_figfile=False) class CoherenceMap(Figure): isConfig = None @@ -647,7 +616,8 @@ class CoherenceMap(Figure): self.HEIGHTPROF = 0 self.counter_imagwr = 0 - self.PLOT_CODE = 3 + self.PLOT_CODE = COH_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -815,27 +785,18 @@ class CoherenceMap(Figure): if x[1] >= self.axesList[0].xmax: self.counter_imagwr = wr_period self.__isConfig = False - - if save: - - if figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 + self.figfile = None - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, figfile) - self.counter_imagwr = 0 - - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime, + update_figfile=False) class PowerProfile(Figure): + isConfig = None __nsubplots = None @@ -847,6 +808,8 @@ class PowerProfile(Figure): self.isConfig = False self.__nsubplots = 1 + self.PLOT_CODE = POWER_CODE + self.WIDTH = 300 self.HEIGHT = 500 self.counter_imagwr = 0 @@ -948,18 +911,12 @@ class PowerProfile(Figure): self.draw() - if save: - - if figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 - - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, figfile) - self.counter_imagwr = 0 + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) class Noise(Figure): @@ -981,7 +938,8 @@ class Noise(Figure): self.xdata = None self.ydata = None - self.PLOT_CODE = 17 + self.PLOT_CODE = NOISE_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -1137,25 +1095,15 @@ class Noise(Figure): del self.xdata del self.ydata self.__isConfig = False - - if save != '': - - if self.figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - self.figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 + self.figfile = None - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, self.figfile) - self.counter_imagwr = 0 - - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime, + update_figfile=False) class BeaconPhase(Figure): @@ -1178,7 +1126,8 @@ class BeaconPhase(Figure): self.xdata = None self.ydata = None - self.PLOT_CODE = 18 + self.PLOT_CODE = BEACON_CODE + self.FTP_WEI = None self.EXP_CODE = None self.SUB_EXP_CODE = None @@ -1354,22 +1303,12 @@ class BeaconPhase(Figure): del self.xdata del self.ydata self.__isConfig = False - - if save: - - if self.figfile == None: - str_datetime = thisDatetime.strftime("%Y%m%d_%H%M%S") - self.figfile = self.getFilename(name = str_datetime) - - self.counter_imagwr += 1 + self.figfile = None - if (self.counter_imagwr>=wr_period): - # store png plot to local folder - self.saveFigure(figpath, self.figfile) - self.counter_imagwr = 0 - - if ftp: - # store png plot to FTP server according to RT-Web format - name = self.getNameToFtp(thisDatetime, self.FTP_WEI, self.EXP_CODE, self.SUB_EXP_CODE, self.PLOT_CODE, self.PLOT_POS) - ftp_filename = os.path.join(figpath, name) - self.saveFigure(figpath, ftp_filename) + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime, + update_figfile=False) diff --git a/schainpy/model/graphics/jroplot_voltage.py b/schainpy/model/graphics/jroplot_voltage.py index 2fcc09e..f1b4ad7 100644 --- a/schainpy/model/graphics/jroplot_voltage.py +++ b/schainpy/model/graphics/jroplot_voltage.py @@ -172,15 +172,9 @@ class Scope(Figure): self.draw() - if save: - date = thisDatetime.strftime("%Y%m%d_%H%M%S") - if figfile == None: - figfile = self.getFilename(name = date) - - self.saveFigure(figpath, figfile) - - self.counter_imagwr += 1 - if (ftp and (self.counter_imagwr==wr_period)): - ftp_filename = os.path.join(figpath,figfile) - self.sendByFTP_Thread(ftp_filename, server, folder, username, password) - self.counter_imagwr = 0 + self.save(figpath=figpath, + figfile=figfile, + save=save, + ftp=ftp, + wr_period=wr_period, + thisDatetime=thisDatetime) diff --git a/schainpy/model/graphics/plotting_codes.py b/schainpy/model/graphics/plotting_codes.py new file mode 100644 index 0000000..5fc8fa8 --- /dev/null +++ b/schainpy/model/graphics/plotting_codes.py @@ -0,0 +1,27 @@ +''' +@author: roj-idl71 +''' +#USED IN jroplot_spectra.py +RTI_CODE = 0 #Range time intensity (RTI). +SPEC_CODE = 1 #Spectra (and Cross-spectra) information. +CROSS_CODE = 2 #Cross-Correlation information. +COH_CODE = 3 #Coherence map. +BASE_CODE = 4 #Base lines graphic. +ROW_CODE = 5 #Row Spectra. +TOTAL_CODE = 6 #Total Power. +DRIFT_CODE = 7 #Drifts graphics. +HEIGHT_CODE = 8 #Height profile. +PHASE_CODE = 9 #Signal Phase. + +POWER_CODE = 16 +NOISE_CODE = 17 +BEACON_CODE = 18 + +#USED IN jroplot_parameters.py + +MOMENTS_CODE = 20 +SKYMAP_CODE = 21 +WIND_CODE = 22 +PARMS_CODE = 23 +SPECFIT_CODE = 24 +EWDRIFT_CODE = 25 diff --git a/schainpy/model/io/jroIO_base.py b/schainpy/model/io/jroIO_base.py index f706f76..7723bf8 100644 --- a/schainpy/model/io/jroIO_base.py +++ b/schainpy/model/io/jroIO_base.py @@ -598,7 +598,7 @@ class JRODataReader(JRODataIO): self.fileSize = fileSize self.fp = fp - print "Setting the file: %s"%self.filename + print "[Reading] Setting the file: %s"%self.filename return 1 @@ -681,14 +681,14 @@ class JRODataReader(JRODataIO): if self.fp != None: self.fp.close() self.fp = open(fullfilename, 'rb') self.flagNoMoreFiles = 0 - print 'Setting the file: %s' % fullfilename + print '[Reading] Setting the file: %s' % fullfilename else: self.fileSize = 0 self.filename = None self.flagIsNewFile = 0 self.fp = None self.flagNoMoreFiles = 1 - print 'No more Files' + print '[Reading] No more files to read' return fileOk_flag @@ -959,7 +959,7 @@ class JRODataReader(JRODataIO): sleep( self.delay ) if not(fullpath): - print "There 'isn't any valid file in %s" % path + print "[Reading] There 'isn't any valid file in %s" % path return None self.year = year diff --git a/schainpy/model/io/jroIO_usrp_api.py b/schainpy/model/io/jroIO_usrp_api.py index ab9c0ca..893b2fe 100644 --- a/schainpy/model/io/jroIO_usrp_api.py +++ b/schainpy/model/io/jroIO_usrp_api.py @@ -105,7 +105,7 @@ class USRPReaderAPI(USRPReader, threading.Thread): def run(self): ''' - This method will be called many times so here you should put all your code + This method will be called once when start() is called ''' if not self.isConfig: diff --git a/schainpy/model/proc/jroproc_base.py b/schainpy/model/proc/jroproc_base.py index 007bd1e..d1922fe 100644 --- a/schainpy/model/proc/jroproc_base.py +++ b/schainpy/model/proc/jroproc_base.py @@ -32,7 +32,7 @@ class ProcessingUnit(object): self.dataIn = None self.dataInList = [] - self.dataOut = {} + self.dataOut = None self.operations2RunDict = {} @@ -108,7 +108,10 @@ class ProcessingUnit(object): # # if name != 'run': # return True - + + if self.dataOut == None: + return False + if self.dataOut.isEmpty(): return False @@ -218,7 +221,11 @@ class ProcessingUnit(object): def run(self): raise ValueError, "Not implemented" - + + def close(self): + #Close every thread, queue or any other object here is it is neccesary. + return + class Operation(object): """ diff --git a/schainpy/model/utils/jroutils_ftp.py b/schainpy/model/utils/jroutils_ftp.py index a5b9ee6..3a5c89a 100644 --- a/schainpy/model/utils/jroutils_ftp.py +++ b/schainpy/model/utils/jroutils_ftp.py @@ -4,9 +4,565 @@ import os import glob import ftplib + +try: + import paramiko + import scp +except: + print "You should install paramiko if you will use SSH protocol to upload files to a server" + import multiprocessing + +import time +import threading + + +try: + from gevent import sleep +except: + from time import sleep + from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation +class Remote(threading.Thread): + """ + Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are + used to upload or download files remotely. + + Non-standard Python modules used: + None + + Written by: + + "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015 + + """ + + server = None + username = None + password = None + remotefolder = None + + period = 60 + fileList = [] + bussy = False + + def __init__(self, server, username, password, remotefolder, period=60): + + threading.Thread.__init__(self) + self._stop = threading.Event() + + self.status = 0 + + self.period = period + self.fileList = [] + self.bussy = False + + self.stopFlag = False + + print "[Remote Server] Opening server: %s" %server + if self.open(server, username, password, remotefolder): + print "[Remote Server] %s server was opened successfully" %server + + def stop(self): + + self.stopFlag = True + + def open(self): + """ + Connect to server and create a connection class (FTP or SSH) to remote server. + """ + raise NotImplementedError, "Implement this method in child class" + + def close(self): + """ + Close connection to server + """ + raise NotImplementedError, "Implement this method in child class" + + def mkdir(self, remotefolder): + """ + Create a folder remotely + """ + raise NotImplementedError, "Implement this method in child class" + + def cd(self, remotefolder): + """ + Change working directory in remote server + """ + raise NotImplementedError, "Implement this method in child class" + + def download(self, filename, localfolder=None): + """ + Download a file from server to local host + """ + raise NotImplementedError, "Implement this method in child class" + + def sendFile(self, fullfilename): + """ + sendFile method is used to upload a local file to the current directory in remote server + + Inputs: + fullfilename - full path name of local file to store in remote directory + + Returns: + 0 in error case else 1 + """ + raise NotImplementedError, "Implement this method in child class" + + def upload(self, fullfilename, remotefolder=None): + """ + upload method is used to upload a local file to remote directory. This method changes + working directory before sending a file. + + Inputs: + fullfilename - full path name of local file to store in remote directory + + remotefolder - remote directory + + Returns: + 0 in error case else 1 + """ + print "[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder) + + if not self.status: + return 0 + + if remotefolder == None: + remotefolder = self.remotefolder + + if not self.cd(remotefolder): + return 0 + + if not self.sendFile(fullfilename): + print "[Remote Server] Error uploading file %s" %fullfilename + return 0 + + print "[Remote Server] upload finished successfully" + + return 1 + + def delete(self, filename): + """ + Remove a file from remote server + """ + pass + + def updateFileList(self, fileList): + """ + Remove a file from remote server + """ + + if fileList == self.fileList: + return 1 + + init = time.time() + + while(self.bussy): + sleep(0.1) + if time.time() - init > 2*self.period: + return 0 + + self.fileList = fileList + + return 1 + + def run(self): + + if not self.cd(self.remotefolder): + raise ValueError, "It could not change to the new remote directory: %s" %remotefolder + + while True: + + sleep(self.period) + + self.bussy = True + + for thisFile in self.fileList: + self.upload(thisFile, self.remotefolder) + + self.bussy = False + + if self.stopFlag: + break + + print "[Remote Server] Thread stopped successfully" + +class FTPClient(Remote): + + __ftpClientObj = None + + def __init__(self, server, username, password, remotefolder, period=60): + """ + """ + Remote.__init__(self, server, username, password, remotefolder, period) + + def open(self, server, username, password, remotefolder): + + """ + This method is used to set FTP parameters and establish a connection to remote server + + Inputs: + server - remote server IP Address + + username - remote server Username + + password - remote server password + + remotefolder - remote server current working directory + + Return: void + + Affects: + self.status - in case of error or fail connection this parameter is set to 0 else 1 + + """ + + if server == None: + raise ValueError, "FTP server should be defined" + + if username == None: + raise ValueError, "FTP username should be defined" + + if password == None: + raise ValueError, "FTP password should be defined" + + if remotefolder == None: + raise ValueError, "FTP remote folder should be defined" + + try: + ftpClientObj = ftplib.FTP(server) + except ftplib.all_errors: + print "FTP server connection fail: %s" %server + self.status = 0 + return 0 + + try: + ftpClientObj.login(username, password) + except ftplib.all_errors: + print "FTP username or password are incorrect" + self.status = 0 + return 0 + + if remotefolder == None: + remotefolder = ftpClientObj.pwd() + else: + try: + ftpClientObj.cwd(remotefolder) + except ftplib.all_errors: + print "FTP remote folder is invalid: %s" %remotefolder + remotefolder = ftpClientObj.pwd() + + self.server = server + self.username = username + self.password = password + self.remotefolder = remotefolder + self.__ftpClientObj = ftpClientObj + self.status = 1 + + return 1 + + def close(self): + """ + Close connection to remote server + """ + if not self.status: + return 0 + + self.__ftpClientObj.close() + + def mkdir(self, remotefolder): + """ + mkdir is used to make a new directory in remote server + + Input: + remotefolder - directory name + + Return: + 0 in error case else 1 + """ + if not self.status: + return 0 + + try: + self.__ftpClientObj.mkd(dirname) + except ftplib.all_errors: + print "Error creating remote folder: %s" %remotefolder + return 0 + + return 1 + + def cd(self, remotefolder): + """ + cd is used to change remote working directory on server + + Input: + remotefolder - current working directory + + Affects: + self.remotefolder + + Return: + 0 in case of error else 1 + """ + if not self.status: + return 0 + + if remotefolder == self.remotefolder: + return 1 + + try: + self.__ftpClientObj.cwd(remotefolder) + except ftplib.all_errors: + print 'Error changing to %s' %remotefolder + print 'Trying to create remote folder' + + if not self.mkdir(remotefolder): + print 'Remote folder could not be created' + return 0 + + try: + self.__ftpClientObj.cwd(remotefolder) + except ftplib.all_errors: + return 0 + + self.remotefolder = remotefolder + + return 1 + + def sendFile(self, fullfilename): + + if not self.status: + return 0 + + file = open(fullfilename, 'rb') + + filename = os.path.split(fullfilename)[-1] + + command = "STOR %s" %filename + + try: + self.__ftpClientObj.storbinary(command, file) + except ftplib.all_errors: + return 0 + + try: + self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename) + except ftplib.all_errors, e: + print e + + file.close() + + return 1 + +class SSHClient(Remote): + + __sshClientObj = None + __scpClientObj = None + + def __init__(self, server, username, password, remotefolder, period=60): + """ + """ + Remote.__init__(self, server, username, password, remotefolder, period) + + def open(self, server, username, password, remotefolder, port=22): + + """ + This method is used to set SSH parameters and establish a connection to a remote server + + Inputs: + server - remote server IP Address + + username - remote server Username + + password - remote server password + + remotefolder - remote server current working directory + + Return: void + + Affects: + self.status - in case of error or fail connection this parameter is set to 0 else 1 + + """ + + if server == None: + raise ValueError, "SSH server should be defined" + + if username == None: + raise ValueError, "SSH username should be defined" + + if password == None: + raise ValueError, "SSH password should be defined" + + if remotefolder == None: + raise ValueError, "SSH remote folder should be defined" + + try: + sshClientObj = paramiko.SSHClient() + except: + print "SSH server connection fail: %s" %server + self.status = 0 + return 0 + + sshClientObj.load_system_host_keys() + sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy()) + + try: + sshClientObj.connect(server, username=username, password=password, port=port) + except : + print "SSH username or password are incorrect: %s" + self.status = 0 + return 0 + + scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30) + + if remotefolder == None: + remotefolder = self.pwd() + + self.server = server + self.username = username + self.password = password + self.remotefolder = remotefolder + self.__sshClientObj = sshClientObj + self.__scpClientObj = scpClientObj + self.status = 1 + + return 1 + + def close(self): + """ + Close connection to remote server + """ + if not self.status: + return 0 + + self.__sshObj.close() + + def mkdir(self, remotefolder): + """ + mkdir is used to make a new directory in remote server + + Input: + remotefolder - directory name + + Return: + 0 in error case else 1 + """ + if not self.status: + return 0 + + stdin, stdout, stderr = self.__sshClientObj.exec_command('mkdir %s' %remotefolder) + result = stderr.readlines()[0] + + if len(result) > 1: + return 0 + + return 1 + + def pwd(self): + + if not self.status: + return None + + stdin, stdout, stderr = self.__sshClientObj.exec_command('pwd') + result = stdout.readlines()[0] + + if len(result) < 1: + return None + + return result[:-1] + + def cd(self, remotefolder): + """ + cd is used to change remote working directory on server + + Input: + remotefolder - current working directory + + Affects: + self.remotefolder + + Return: + 0 in case of error else 1 + """ + if not self.status: + return 0 + + if remotefolder == self.remotefolder: + return 1 + + self.remotefolder = remotefolder + + return 1 + + def sendFile(self, fullfilename): + + if not self.status: + return 0 + + try: + self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder) + except: + return 0 + + return 1 + +class SendToServer(ProcessingUnit): + + def __init__(self): + + ProcessingUnit.__init__(self) + + self.isConfig = False + self.clientObj = None + + def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp'): + + self.clientObj = None + self.localfolder = localfolder + self.ext = ext + self.period = period + + if str.lower(protocol) == 'ftp': + self.clientObj = FTPClient(server, username, password, remotefolder, period) + + if str.lower(protocol) == 'ssh': + self.clientObj = SSHClient(server, username, password, remotefolder, period) + + if not self.clientObj: + raise ValueError, "%s has been chosen as remote access protocol but it is not valid" %protocol + + self.clientObj.start() + + def findFiles(self): + + filenameList = glob.glob1(self.localfolder, '*%s' %self.ext) + + if len(filenameList) < 1: + return [] + + fullfilenameList = [os.path.join(self.localfolder, thisFile) for thisFile in filenameList] + + return fullfilenameList + + def run(self, **kwargs): + + if not self.isConfig: + self.init = time.time() + self.setup(**kwargs) + self.isConfig = True + + if time.time() - self.init >= self.period: + fullfilenameList = self.findFiles() + self.clientObj.updateFileList(fullfilenameList) + self.init = time.time() + + def close(self): + print "[Remote Server] Stopping thread" + self.clientObj.stop() + + class FTP(object): """ Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module @@ -298,20 +854,23 @@ class FTP(object): self.ftp.close() class SendByFTP(Operation): + def __init__(self): + self.status = 1 self.counter = 0 def error_print(self, ValueError): + print ValueError, 'Error FTP' print "don't worry the program is running..." def worker_ftp(self, server, username, password, remotefolder, filenameList): - self.ftpObj = FTP(server, username, password, remotefolder) + self.ftpClientObj = FTP(server, username, password, remotefolder) for filename in filenameList: - self.ftpObj.upload(filename) - self.ftpObj.close() + self.ftpClientObj.upload(filename) + self.ftpClientObj.close() def ftp_thread(self, server, username, password, remotefolder): if not(self.status): diff --git a/schainpy/model/utils/scp.py b/schainpy/model/utils/scp.py new file mode 100644 index 0000000..734acb7 --- /dev/null +++ b/schainpy/model/utils/scp.py @@ -0,0 +1,464 @@ +# scp.py +# Copyright (C) 2008 James Bardin + +""" +Utilities for sending files over ssh using the scp1 protocol. +""" + +__version__ = '0.10.0' + +import locale +import os +import re +from socket import timeout as SocketTimeout + + +# this is quote from the shlex module, added in py3.3 +_find_unsafe = re.compile(br'[^\w@%+=:,./~-]').search + + +def _sh_quote(s): + """Return a shell-escaped version of the string `s`.""" + if not s: + return b"" + if _find_unsafe(s) is None: + return s + + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return b"'" + s.replace(b"'", b"'\"'\"'") + b"'" + + +# Unicode conversion functions; assume UTF-8 + +def asbytes(s): + """Turns unicode into bytes, if needed. + Assumes UTF-8. + """ + if isinstance(s, bytes): + return s + else: + return s.encode('utf-8') + + +def asunicode(s): + """Turns bytes into unicode, if needed. + Uses UTF-8. + """ + if isinstance(s, bytes): + return s.decode('utf-8', 'replace') + else: + return s + + +# os.path.sep is unicode on Python 3, no matter the platform +bytes_sep = asbytes(os.path.sep) + + +# Unicode conversion function for Windows +# Used to convert local paths if the local machine is Windows + +def asunicode_win(s): + """Turns bytes into unicode, if needed. + """ + if isinstance(s, bytes): + return s.decode(locale.getpreferredencoding()) + else: + return s + + +class SCPClient(object): + """ + An scp1 implementation, compatible with openssh scp. + Raises SCPException for all transport related errors. Local filesystem + and OS errors pass through. + Main public methods are .put and .get + The get method is controlled by the remote scp instance, and behaves + accordingly. This means that symlinks are resolved, and the transfer is + halted after too many levels of symlinks are detected. + The put method uses os.walk for recursion, and sends files accordingly. + Since scp doesn't support symlinks, we send file symlinks as the file + (matching scp behaviour), but we make no attempt at symlinked directories. + """ + def __init__(self, transport, buff_size=16384, socket_timeout=5.0, + progress=None, sanitize=_sh_quote): + """ + Create an scp1 client. + @param transport: an existing paramiko L{Transport} + @type transport: L{Transport} + @param buff_size: size of the scp send buffer. + @type buff_size: int + @param socket_timeout: channel socket timeout in seconds + @type socket_timeout: float + @param progress: callback - called with (filename, size, sent) during + transfers + @param sanitize: function - called with filename, should return + safe or escaped string. Uses _sh_quote by default. + @type progress: function(string, int, int) + """ + self.transport = transport + self.buff_size = buff_size + self.socket_timeout = socket_timeout + self.channel = None + self.preserve_times = False + self._progress = progress + self._recv_dir = b'' + self._rename = False + self._utime = None + self.sanitize = sanitize + self._dirtimes = {} + + def __enter__(self): + self.channel = self._open() + return self + + def __exit__(self, type, value, traceback): + self.close() + + def put(self, files, remote_path=b'.', + recursive=False, preserve_times=False): + """ + Transfer files to remote host. + @param files: A single path, or a list of paths to be transfered. + recursive must be True to transfer directories. + @type files: string OR list of strings + @param remote_path: path in which to receive the files on the remote + host. defaults to '.' + @type remote_path: str + @param recursive: transfer files and directories recursively + @type recursive: bool + @param preserve_times: preserve mtime and atime of transfered files + and directories. + @type preserve_times: bool + """ + self.preserve_times = preserve_times + self.channel = self._open() + self._pushed = 0 + self.channel.settimeout(self.socket_timeout) + scp_command = (b'scp -t ', b'scp -r -t ')[recursive] + self.channel.exec_command(scp_command + + self.sanitize(asbytes(remote_path))) + self._recv_confirm() + + if not isinstance(files, (list, tuple)): + files = [files] + + if recursive: + self._send_recursive(files) + else: + self._send_files(files) + + self.close() + + def get(self, remote_path, local_path='', + recursive=False, preserve_times=False): + """ + Transfer files from remote host to localhost + @param remote_path: path to retreive from remote host. since this is + evaluated by scp on the remote host, shell wildcards and + environment variables may be used. + @type remote_path: str + @param local_path: path in which to receive files locally + @type local_path: str + @param recursive: transfer files and directories recursively + @type recursive: bool + @param preserve_times: preserve mtime and atime of transfered files + and directories. + @type preserve_times: bool + """ + if not isinstance(remote_path, (list, tuple)): + remote_path = [remote_path] + remote_path = [self.sanitize(asbytes(r)) for r in remote_path] + self._recv_dir = local_path or os.getcwd() + self._rename = (len(remote_path) == 1 and + not os.path.isdir(os.path.abspath(local_path))) + if len(remote_path) > 1: + if not os.path.exists(self._recv_dir): + raise SCPException("Local path '%s' does not exist" % + asunicode(self._recv_dir)) + elif not os.path.isdir(self._recv_dir): + raise SCPException("Local path '%s' is not a directory" % + asunicode(self._recv_dir)) + rcsv = (b'', b' -r')[recursive] + prsv = (b'', b' -p')[preserve_times] + self.channel = self._open() + self._pushed = 0 + self.channel.settimeout(self.socket_timeout) + self.channel.exec_command(b"scp" + + rcsv + + prsv + + b" -f " + + b' '.join(remote_path)) + self._recv_all() + self.close() + + def _open(self): + """open a scp channel""" + if self.channel is None: + self.channel = self.transport.open_session() + + return self.channel + + def close(self): + """close scp channel""" + if self.channel is not None: + self.channel.close() + self.channel = None + + def _read_stats(self, name): + """return just the file stats needed for scp""" + if os.name == 'nt': + name = asunicode(name) + stats = os.stat(name) + mode = oct(stats.st_mode)[-4:] + size = stats.st_size + atime = int(stats.st_atime) + mtime = int(stats.st_mtime) + return (mode, size, mtime, atime) + + def _send_files(self, files): + for name in files: + basename = asbytes(os.path.basename(name)) + (mode, size, mtime, atime) = self._read_stats(name) + if self.preserve_times: + self._send_time(mtime, atime) + file_hdl = open(name, 'rb') + + # The protocol can't handle \n in the filename. + # Quote them as the control sequence \^J for now, + # which is how openssh handles it. + self.channel.sendall(("C%s %d " % (mode, size)).encode('ascii') + + basename.replace(b'\n', b'\\^J') + b"\n") + self._recv_confirm() + file_pos = 0 + if self._progress: + if size == 0: + # avoid divide-by-zero + self._progress(basename, 1, 1) + else: + self._progress(basename, size, 0) + buff_size = self.buff_size + chan = self.channel + while file_pos < size: + chan.sendall(file_hdl.read(buff_size)) + file_pos = file_hdl.tell() + if self._progress: + self._progress(basename, size, file_pos) + chan.sendall('\x00') + file_hdl.close() + self._recv_confirm() + + def _chdir(self, from_dir, to_dir): + # Pop until we're one level up from our next push. + # Push *once* into to_dir. + # This is dependent on the depth-first traversal from os.walk + + # add path.sep to each when checking the prefix, so we can use + # path.dirname after + common = os.path.commonprefix([from_dir + bytes_sep, + to_dir + bytes_sep]) + # now take the dirname, since commonprefix is character based, + # and we either have a seperator, or a partial name + common = os.path.dirname(common) + cur_dir = from_dir.rstrip(bytes_sep) + while cur_dir != common: + cur_dir = os.path.split(cur_dir)[0] + self._send_popd() + # now we're in our common base directory, so on + self._send_pushd(to_dir) + + def _send_recursive(self, files): + for base in files: + if not os.path.isdir(base): + # filename mixed into the bunch + self._send_files([base]) + continue + last_dir = asbytes(base) + for root, dirs, fls in os.walk(base): + self._chdir(last_dir, asbytes(root)) + self._send_files([os.path.join(root, f) for f in fls]) + last_dir = asbytes(root) + # back out of the directory + while self._pushed > 0: + self._send_popd() + + def _send_pushd(self, directory): + (mode, size, mtime, atime) = self._read_stats(directory) + basename = asbytes(os.path.basename(directory)) + if self.preserve_times: + self._send_time(mtime, atime) + self.channel.sendall(('D%s 0 ' % mode).encode('ascii') + + basename.replace(b'\n', b'\\^J') + b'\n') + self._recv_confirm() + self._pushed += 1 + + def _send_popd(self): + self.channel.sendall('E\n') + self._recv_confirm() + self._pushed -= 1 + + def _send_time(self, mtime, atime): + self.channel.sendall(('T%d 0 %d 0\n' % (mtime, atime)).encode('ascii')) + self._recv_confirm() + + def _recv_confirm(self): + # read scp response + msg = b'' + try: + msg = self.channel.recv(512) + except SocketTimeout: + raise SCPException('Timout waiting for scp response') + # slice off the first byte, so this compare will work in py2 and py3 + if msg and msg[0:1] == b'\x00': + return + elif msg and msg[0:1] == b'\x01': + raise SCPException(asunicode(msg[1:])) + elif self.channel.recv_stderr_ready(): + msg = self.channel.recv_stderr(512) + raise SCPException(asunicode(msg)) + elif not msg: + raise SCPException('No response from server') + else: + raise SCPException('Invalid response from server', msg) + + def _recv_all(self): + # loop over scp commands, and receive as necessary + command = {b'C': self._recv_file, + b'T': self._set_time, + b'D': self._recv_pushd, + b'E': self._recv_popd} + while not self.channel.closed: + # wait for command as long as we're open + self.channel.sendall('\x00') + msg = self.channel.recv(1024) + if not msg: # chan closed while recving + break + assert msg[-1:] == b'\n' + msg = msg[:-1] + code = msg[0:1] + try: + command[code](msg[1:]) + except KeyError: + raise SCPException(asunicode(msg[1:])) + # directory times can't be set until we're done writing files + self._set_dirtimes() + + def _set_time(self, cmd): + try: + times = cmd.split(b' ') + mtime = int(times[0]) + atime = int(times[2]) or mtime + except: + self.channel.send(b'\x01') + raise SCPException('Bad time format') + # save for later + self._utime = (atime, mtime) + + def _recv_file(self, cmd): + chan = self.channel + parts = cmd.strip().split(b' ', 2) + + try: + mode = int(parts[0], 8) + size = int(parts[1]) + if self._rename: + path = self._recv_dir + self._rename = False + elif os.name == 'nt': + path = os.path.join(asunicode_win(self._recv_dir), + parts[2].decode('utf-8')) + else: + path = os.path.join(asbytes(self._recv_dir), + parts[2]) + except: + chan.send('\x01') + chan.close() + raise SCPException('Bad file format') + + try: + file_hdl = open(path, 'wb') + except IOError as e: + chan.send(b'\x01' + str(e).encode('utf-8')) + chan.close() + raise + + if self._progress: + if size == 0: + # avoid divide-by-zero + self._progress(path, 1, 1) + else: + self._progress(path, size, 0) + buff_size = self.buff_size + pos = 0 + chan.send(b'\x00') + try: + while pos < size: + # we have to make sure we don't read the final byte + if size - pos <= buff_size: + buff_size = size - pos + file_hdl.write(chan.recv(buff_size)) + pos = file_hdl.tell() + if self._progress: + self._progress(path, size, pos) + + msg = chan.recv(512) + if msg and msg[0:1] != b'\x00': + raise SCPException(asunicode(msg[1:])) + except SocketTimeout: + chan.close() + raise SCPException('Error receiving, socket.timeout') + + file_hdl.truncate() + try: + os.utime(path, self._utime) + self._utime = None + os.chmod(path, mode) + # should we notify the other end? + finally: + file_hdl.close() + # '\x00' confirmation sent in _recv_all + + def _recv_pushd(self, cmd): + parts = cmd.split(b' ', 2) + try: + mode = int(parts[0], 8) + if self._rename: + path = self._recv_dir + self._rename = False + elif os.name == 'nt': + path = os.path.join(asunicode_win(self._recv_dir), + parts[2].decode('utf-8')) + else: + path = os.path.join(asbytes(self._recv_dir), + parts[2]) + except: + self.channel.send(b'\x01') + raise SCPException('Bad directory format') + try: + if not os.path.exists(path): + os.mkdir(path, mode) + elif os.path.isdir(path): + os.chmod(path, mode) + else: + raise SCPException('%s: Not a directory' % path) + self._dirtimes[path] = (self._utime) + self._utime = None + self._recv_dir = path + except (OSError, SCPException) as e: + self.channel.send(b'\x01' + asbytes(str(e))) + raise + + def _recv_popd(self, *cmd): + self._recv_dir = os.path.split(self._recv_dir)[0] + + def _set_dirtimes(self): + try: + for d in self._dirtimes: + os.utime(d, self._dirtimes[d]) + finally: + self._dirtimes = {} + + +class SCPException(Exception): + """SCP exception class""" + pass \ No newline at end of file