schain Commit - r1284:f79a273a3a9b · Jicamarca Repository

Fix CLI and remove imports warnings

Juan C. Espinoza -

r1284:f79a273a3a9b

parent child

Context file:

r1284:f79a273a3a9b

Collapse all files

schainpy/cli/cli.py +13 -8

             import click
             import schainpy
             import subprocess
             import os
             import sys
             import glob
-            save_stdout = sys.stdout
-            sys.stdout = open('/dev/null', 'w')
             from multiprocessing import cpu_count
             from schainpy.controller import Project
             from schainpy.model import Operation, ProcessingUnit
             from schainpy.utils import log
             from importlib import import_module
             from pydoc import locate
             from fuzzywuzzy import process
             from schainpy.cli import templates
             import inspect
             try:
                 from queue import Queue
             except:
                 from Queue import Queue
-            sys.stdout = save_stdout
             def getProcs():
                 modules = dir(schainpy.model)
                 procs = check_module(modules, 'processing')
                 try:
                     procs.remove('ProcessingUnit')
                 except Exception as e:
                     pass
                 return procs
             def getOperations():
                 module = dir(schainpy.model)
                 noProcs = [x for x in module if not x.endswith('Proc')]
                 operations = check_module(noProcs, 'operation')
                 try:
                     operations.remove('Operation')
                     operations.remove('Figure')
                     operations.remove('Plot')
                 except Exception as e:
                     pass
                 return operations
             def getArgs(op):
                 module = locate('schainpy.model.{}'.format(op))
+                try:
-                if hasattr(module, '__attrs__'):
+                    obj = module(1,2,3,Queue(),5,6)
-                    args = module.__attrs__
+                except:
+                    obj = module()
+                if hasattr(obj, '__attrs__'):
+                    args = obj.__attrs__
                 else:
-                    args = inspect.getargspec(module.run).args
+                    if hasattr(obj, 'myrun'):
+                        args = inspect.getfullargspec(obj.myrun).args
+                    else:
+                        args = inspect.getfullargspec(obj.run).args
                 try:
                     args.remove('self')
                 except Exception as e:
                     pass
                 try:
                     args.remove('dataOut')
                 except Exception as e:
                     pass
                 return args
             def getDoc(obj):
                 module = locate('schainpy.model.{}'.format(obj))
                 try:
                     obj = module(1,2,3,Queue(),5,6)
                 except:
                     obj = module()
                 return obj.__doc__
             def getAll():
                 modules = getOperations()
                 modules.extend(getProcs())
                 return modules
             def print_version(ctx, param, value):
                 if not value or ctx.resilient_parsing:
                     return
                 click.echo(schainpy.__version__)
                 ctx.exit()
             PREFIX = 'experiment'
             @click.command()
             @click.option('--version', '-v', is_flag=True, callback=print_version, help='SChain version', type=str)
             @click.argument('command', default='run', required=True)
             @click.argument('nextcommand', default=None, required=False, type=str)
             def main(command, nextcommand, version):
                 """COMMAND LINE INTERFACE FOR SIGNAL CHAIN - JICAMARCA RADIO OBSERVATORY V3.0\n
                     Available commands.\n
                     xml: runs a schain XML generated file\n
                     run: runs any python script starting 'experiment_'\n
                     generate: generates a template schain script\n
                     list: return a list of available procs and operations\n
                     search: return avilable operations, procs or arguments of the given
                             operation/proc\n"""
                 if command == 'xml':
                     runFromXML(nextcommand)
                 elif command == 'generate':
                     generate()
                 elif command == 'test':
                     test()
                 elif command == 'run':
                     runschain(nextcommand)
                 elif command == 'search':
                     search(nextcommand)
                 elif command == 'list':
                     cmdlist(nextcommand)
                 else:
                     log.error('Command {} is not defined'.format(command))
             def check_module(possible, instance):
                 def check(x):
                     try:
                         instancia = locate('schainpy.model.{}'.format(x))
                         ret = instancia.proc_type == instance
                         return ret
                     except Exception as e:
                         return False
                 clean = clean_modules(possible)
                 return [x for x in clean if check(x)]
             def clean_modules(module):
                 noEndsUnder = [x for x in module if not x.endswith('__')]
                 noStartUnder = [x for x in noEndsUnder if not x.startswith('__')]
                 noFullUpper = [x for x in noStartUnder if not x.isupper()]
                 return noFullUpper
             def cmdlist(nextcommand):
                 if nextcommand is None:
                     log.error('Missing argument, available arguments: procs, operations', '')
                 elif nextcommand == 'procs':
                     procs = getProcs()
                     log.success(
                         'Current ProcessingUnits are:\n  {}'.format('\n  '.join(procs)), '')
                 elif nextcommand == 'operations':
                     operations = getOperations()
                     log.success('Current Operations are:\n  {}'.format(
                         '\n  '.join(operations)), '')
                 else:
                     log.error('Wrong argument', '')
             def search(nextcommand):
                 if nextcommand is None:
                     log.error('There is no Operation/ProcessingUnit to search', '')
                 else:
                     try:
                         args = getArgs(nextcommand)
                         doc = getDoc(nextcommand)
                         if len(args) == 0:
                             log.success('\n{} has no arguments'.format(nextcommand), '')
                         else:
                             log.success('{}\n{}\n\narguments:\n  {}'.format(
                                 nextcommand, doc, ', '.join(args)), '')
                     except Exception as e:
                         log.error('Module `{}` does not exists'.format(nextcommand), '')
                         allModules = getAll()
                         similar = [t[0] for t in process.extract(nextcommand, allModules, limit=12) if t[1]>80]
                         log.success('Possible modules are: {}'.format(', '.join(similar)), '')
             def runschain(nextcommand):
                 if nextcommand is None:
                     currentfiles = glob.glob('./{}_*.py'.format(PREFIX))
                     numberfiles = len(currentfiles)
                     if numberfiles > 1:
                         log.error('There is more than one file to run')
                     elif numberfiles == 1:
                         subprocess.call(['python ' + currentfiles[0]], shell=True)
                     else:
                         log.error('There is no file to run')
                 else:
                     try:
                         subprocess.call(['python ' + nextcommand], shell=True)
                     except Exception as e:
                         log.error("I cannot run the file. Does it exists?")
             def basicInputs():
                 inputs = {}
                 inputs['name'] = click.prompt(
                     'Name of the project', default="project", type=str)
                 inputs['desc'] = click.prompt(
                     'Enter a description', default="A schain project", type=str)
                 inputs['multiprocess'] = click.prompt(
                     '''Select data type:
                 - Voltage (*.r):                [1]
                 - Spectra (*.pdata):            [2]
                 - Voltage and Spectra (*.r):    [3]
                 -->''', type=int)
                 inputs['path'] = click.prompt('Data path', default=os.getcwd(
                 ), type=click.Path(exists=True, resolve_path=True))
                 inputs['startDate'] = click.prompt(
                     'Start date', default='1970/01/01', type=str)
                 inputs['endDate'] = click.prompt(
                     'End date', default='2018/12/31', type=str)
                 inputs['startHour'] = click.prompt(
                     'Start hour', default='00:00:00', type=str)
                 inputs['endHour'] = click.prompt('End hour', default='23:59:59', type=str)
                 inputs['figpath'] = inputs['path'] + '/figs'
                 return inputs
             def generate():
                 inputs = basicInputs()
                 if inputs['multiprocess'] == 1:
                     current = templates.voltage.format(**inputs)
                 elif inputs['multiprocess'] == 2:
                     current = templates.spectra.format(**inputs)
                 elif inputs['multiprocess'] == 3:
                     current = templates.voltagespectra.format(**inputs)
                 scriptname = '{}_{}.py'.format(PREFIX, inputs['name'])
                 script = open(scriptname, 'w')
                 try:
                     script.write(current)
                     log.success('Script {} generated'.format(scriptname))
                 except Exception as e:
                     log.error('I cannot create the file. Do you have writing permissions?')
             def test():
                 log.warning('testing')
             def runFromXML(filename):
                 controller = Project()
                 if not controller.readXml(filename):
                     return
                 controller.start()
                 return

schainpy/model/io/jroIO_digitalRF.py +1 -1

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             # SUBCHANNELS EN VEZ DE CHANNELS
             # BENCHMARKS -> PROBLEMAS CON ARCHIVOS GRANDES -> INCONSTANTE EN EL TIEMPO
             # ACTUALIZACION DE VERSION
             # HEADERS
             # MODULO DE ESCRITURA
             # METADATA
             import os
             import time
             import datetime
             import numpy
             import timeit
             from fractions import Fraction
             from time import time
             from time import sleep
             import schainpy.admin
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             import pickle
             try:
                 import digital_rf
             except:
-                print('You should install "digital_rf" module if you want to read Digital RF data')
+                pass
             @MPDecorator
             class DigitalRFReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self)
                     self.dataOut                  = Voltage()
                     self.__printInfo              = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__codeType    = 0
                     self.__ippKm       = None
                     self.__nCode       = None
                     self.__nBaud       = None
                     self.__code        = None
                     self.dtype         = None
                     self.oldAverage    = None
                     self.path          = None
                 def close(self):
                     print('Average of writing to digital rf format is ', self.oldAverage * 1000)
                     return
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample / self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     ippSeconds = 1.0 * self.__nSamples / self.__sample_rate
                     nProfiles = 1.0 / ippSeconds  # Number of profiles in one second
                     try:
                         self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
                             self.__radarControllerHeader)
                     except:
                         self.dataOut.radarControllerHeaderObj = RadarControllerHeader(
                             txA=0,
                             txB=0,
                             nWindows=1,
                             nHeights=self.__nSamples,
                             firstHeight=self.__firstHeigth,
                             deltaHeight=self.__deltaHeigth,
                             codeType=self.__codeType,
                             nCode=self.__nCode, nBaud=self.__nBaud,
                             code=self.__code)
                     try:
                         self.dataOut.systemHeaderObj = SystemHeader(self.__systemHeader)
                     except:
                         self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                     nProfiles=nProfiles,
                                                                     nChannels=len(
                                                                         self.__channelList),
                                                                     adcResolution=14)
                     self.dataOut.type  = "Voltage"
                     self.dataOut.data  = None
                     self.dataOut.dtype = self.dtype
                     # self.dataOut.nChannels = 0
                     # self.dataOut.nHeights = 0
                     self.dataOut.nProfiles   = int(nProfiles)
                     self.dataOut.heightList  = self.__firstHeigth + \
                         numpy.arange(self.__nSamples, dtype=numpy.float) * \
                         self.__deltaHeigth
                     self.dataOut.channelList = list(range(self.__num_subchannels))
                     self.dataOut.blocksize   = self.dataOut.getNChannels() * self.dataOut.getNHeights()
                     # self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData      = True
                     self.dataOut.flagDataAsBlock = False
                     # Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime     = None
                     # timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.timeZone    = self.__timezone / 60
                     self.dataOut.dstFlag     = 0
                     self.dataOut.errorCount  = 0
                     try:
                         self.dataOut.nCohInt = self.fixed_metadata_dict.get(
                             'nCohInt', self.nCohInt)
                         # asumo que la data esta decodificada
                         self.dataOut.flagDecodeData = self.fixed_metadata_dict.get(
                             'flagDecodeData', self.flagDecodeData)
                         # asumo que la data esta sin flip
                         self.dataOut.flagDeflipData = self.fixed_metadata_dict['flagDeflipData']
                         self.dataOut.flagShiftFFT   = self.fixed_metadata_dict['flagShiftFFT']
                         self.dataOut.useLocalTime   = self.fixed_metadata_dict['useLocalTime']
                     except:
                         pass
                     self.dataOut.ippSeconds = ippSeconds
                     # Time interval between profiles
                     # self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency  = self.__frequency
                     self.dataOut.realtime   = self.__online
                 def findDatafiles(self, path, startDate=None, endDate=None):
                     if not os.path.isdir(path):
                         return []
                     try:
                         digitalReadObj = digital_rf.DigitalRFReader(
                             path, load_all_metadata=True)
                     except:
                         digitalReadObj = digital_rf.DigitalRFReader(path)
                     channelNameList    = digitalReadObj.get_channels()
                     if not channelNameList:
                         return []
                     metadata_dict      = digitalReadObj.get_rf_file_metadata(channelNameList[0])
                     sample_rate        = metadata_dict['sample_rate'][0]
                     this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
                     try:
                         timezone       = this_metadata_file['timezone'].value
                     except:
                         timezone       = 0
                     startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(
                         channelNameList[0]) / sample_rate - timezone
                     startDatetime      = datetime.datetime.utcfromtimestamp(startUTCSecond)
                     endDatatime        = datetime.datetime.utcfromtimestamp(endUTCSecond)
                     if not startDate:
                         startDate      = startDatetime.date()
                     if not endDate:
                         endDate        = endDatatime.date()
                     dateList           = []
                     thisDatetime       = startDatetime
                     while(thisDatetime <= endDatatime):
                         thisDate        = thisDatetime.date()
                         if thisDate     < startDate:
                             continue
                         if thisDate     > endDate:
                             break
                         dateList.append(thisDate)
                         thisDatetime += datetime.timedelta(1)
                     return dateList
                 def setup(self, path=None,
                           startDate=None,
                           endDate=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           channelList=None,
                           nSamples=None,
                           online=False,
                           delay=60,
                           buffer_size=1024,
                           ippKm=None,
                           nCohInt=1,
                           nCode=1,
                           nBaud=1,
                           flagDecodeData=False,
                           code=numpy.ones((1, 1), dtype=numpy.int),
                           **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
                         delay
                     '''
                     self.path           = path
                     self.nCohInt        = nCohInt
                     self.flagDecodeData = flagDecodeData
                     self.i              = 0
                     if not os.path.isdir(path):
                         raise ValueError("[Reading] Directory %s does not exist" % path)
                     try:
                         self.digitalReadObj = digital_rf.DigitalRFReader(
                             path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf.DigitalRFReader(path)
                     channelNameList         = self.digitalReadObj.get_channels()
                     if not channelNameList:
                         raise ValueError("[Reading] Directory %s does not have any files" % path)
                     if not channelList:
                         channelList = list(range(len(channelNameList)))
                     ##########  Reading metadata ######################
                     top_properties           = self.digitalReadObj.get_properties(
                         channelNameList[channelList[0]])
                     self.__num_subchannels   = top_properties['num_subchannels']
                     self.__sample_rate       = 1.0 * \
                         top_properties['sample_rate_numerator'] / \
                         top_properties['sample_rate_denominator']
                     # self.__samples_per_file = top_properties['samples_per_file'][0]
                     self.__deltaHeigth       = 1e6 * 0.15 / self.__sample_rate  # why 0.15?
                     this_metadata_file       = self.digitalReadObj.get_digital_metadata(
                         channelNameList[channelList[0]])
                     metadata_bounds          = this_metadata_file.get_bounds()
                     self.fixed_metadata_dict = this_metadata_file.read(
                         metadata_bounds[0])[metadata_bounds[0]]  # GET FIRST HEADER
                     try:
                         self.__processingHeader      = self.fixed_metadata_dict['processingHeader']
                         self.__radarControllerHeader = self.fixed_metadata_dict['radarControllerHeader']
                         self.__systemHeader          = self.fixed_metadata_dict['systemHeader']
                         self.dtype                   = pickle.loads(self.fixed_metadata_dict['dtype'])
                     except:
                         pass
                     self.__frequency = None
                     self.__frequency = self.fixed_metadata_dict.get('frequency', 1)
                     self.__timezone = self.fixed_metadata_dict.get('timezone', 18000)
                     try:
                         nSamples = self.fixed_metadata_dict['nSamples']
                     except:
                         nSamples = None
                     self.__firstHeigth = 0
                     try:
                         codeType       = self.__radarControllerHeader['codeType']
                     except:
                         codeType       = 0
                     try:
                         if codeType:
                             nCode = self.__radarControllerHeader['nCode']
                             nBaud = self.__radarControllerHeader['nBaud']
                             code  = self.__radarControllerHeader['code']
                     except:
                         pass
                     if not ippKm:
                         try:
                             # seconds to km
                             ippKm = self.__radarControllerHeader['ipp']
                         except:
                             ippKm = None
                     ####################################################
                     self.__ippKm   = ippKm
                     startUTCSecond = None
                     endUTCSecond   = None
                     if startDate:
                         startDatetime  = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (
                             startDatetime - datetime.datetime(1970, 1, 1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime   = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond  = (endDatetime - datetime.datetime(1970,
 , 1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(
                         channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index / self.__sample_rate
                     if start_index     > startUTCSecond * self.__sample_rate:
                         startUTCSecond = start_index / self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond   = end_index / self.__sample_rate
                     if end_index       < endUTCSecond * self.__sample_rate:
                         endUTCSecond   = end_index / self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError("[Reading] nSamples or ippKm should be defined")
                         nSamples            = int(ippKm / (1e6 * 0.15 / self.__sample_rate))
                     channelBoundList        = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName        = channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(
                             thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
                     self.i            = 0
                     self.__delay      = delay
                     self.__codeType   = codeType
                     self.__nCode      = nCode
                     self.__nBaud      = nBaud
                     self.__code       = code
                     self.__datapath         = path
                     self.__online           = online
                     self.__channelList      = channelList
                     self.__channelNameList  = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples         = nSamples
                     self.__samples_to_read  = int(nSamples)  # FIJO: AHORA 40
                     self.__nChannels        = len(self.__channelList)
                     self.__startUTCSecond   = startUTCSecond
                     self.__endUTCSecond     = endUTCSecond
                     self.__timeInterval     = 1.0 * self.__samples_to_read / \
                         self.__sample_rate  # Time interval
                     if online:
                         # self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     # por que en el otro metodo lo primero q se hace es sumar samplestoread
                     self.__thisUnixSample = int(startUTCSecond * self.__sample_rate) - self.__samples_to_read
                     self.__data_buffer    = numpy.zeros(
                         (self.__num_subchannels, self.__samples_to_read), dtype=numpy.complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print("[Reading] Digital RF Data was found from %s to %s " % (
                         datetime.datetime.utcfromtimestamp(
                             self.__startUTCSecond - self.__timezone),
                         datetime.datetime.utcfromtimestamp(
                             self.__endUTCSecond - self.__timezone)
                     ))
                     print("[Reading] Starting process from %s to %s" % (datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
                                                                         datetime.datetime.utcfromtimestamp(
                         endUTCSecond - self.__timezone)
                     ))
                     self.oldAverage    = None
                     self.count         = 0
                     self.executionTime = 0
                 def __reload(self):
                     #         print
                     #         print "%s not in range [%s, %s]" %(
                     #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                     #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                     #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                     #                                           )
                     print("[Reading] reloading metadata ...")
                     try:
                         self.digitalReadObj.reload(complete_update=True)
                     except:
                         self.digitalReadObj = digital_rf.DigitalRFReader(self.path)
                     start_index, end_index  = self.digitalReadObj.get_bounds(
                         self.__channelNameList[self.__channelList[0]])
                     if start_index          > self.__startUTCSecond * self.__sample_rate:
                         self.__startUTCSecond = 1.0 * start_index / self.__sample_rate
                     if end_index            > self.__endUTCSecond * self.__sample_rate:
                         self.__endUTCSecond = 1.0 * end_index / self.__sample_rate
                         print()
                         print("[Reading] New timerange found [%s, %s] " % (
                             datetime.datetime.utcfromtimestamp(
                                 self.__startUTCSecond - self.__timezone),
                             datetime.datetime.utcfromtimestamp(
                                 self.__endUTCSecond - self.__timezone)
                         ))
                         return True
                     return False
                 def timeit(self, toExecute):
                     t0                  = time.time()
                     toExecute()
                     self.executionTime  = time.time() - t0
                     if self.oldAverage is None:
                         self.oldAverage = self.executionTime
                     self.oldAverage     = (self.executionTime + self.count *
                                        self.oldAverage) / (self.count + 1.0)
                     self.count          = self.count + 1.0
                     return
                 def __readNextBlock(self, seconds=30, volt_scale=1):
                     '''
                     '''
                     # Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample        += self.__samples_to_read
                     if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                         print ("[Reading] There are no more data into selected time-range")
                         if self.__online:
                             sleep(3)
                             self.__reload()
                         else:
                             return False
                         if self.__thisUnixSample + 2 * self.__samples_to_read > self.__endUTCSecond * self.__sample_rate:
                             return False
                             self.__thisUnixSample -= self.__samples_to_read
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:  # TODO VARIOS CHANNELS?
                         for indexSubchannel in range(self.__num_subchannels):
                             try:
                                 t0     = time()
                                 result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                               self.__samples_to_read,
                                                                               thisChannelName, sub_channel=indexSubchannel)
                                 self.executionTime  = time() - t0
                                 if self.oldAverage is None:
                                     self.oldAverage = self.executionTime
                                 self.oldAverage     = (
                                     self.executionTime + self.count * self.oldAverage) / (self.count + 1.0)
                                 self.count = self.count + 1.0
                             except IOError as e:
                                 # read next profile
                                 self.__flagDiscontinuousBlock = True
                                 print("[Reading] %s" % datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
                                 break
                             if result.shape[0] != self.__samples_to_read:
                                 self.__flagDiscontinuousBlock = True
                                 print("[Reading] %s: Too few samples were found, just %d/%d  samples" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                                          result.shape[0],
                                                                                                          self.__samples_to_read))
                                 break
                             self.__data_buffer[indexSubchannel, :] = result * volt_scale
                             indexChannel+=1
                             dataOk       = True
                     self.__utctime       = self.__thisUnixSample / self.__sample_rate
                     if not dataOk:
                         return False
                     print("[Reading] %s: %d samples <> %f sec" % (datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                   self.__samples_to_read,
                                                                   self.__timeInterval))
                     self.__bufferIndex   = 0
                     return True
                 def __isBufferEmpty(self):
                     return self.__bufferIndex > self.__samples_to_read - self.__nSamples  # 40960 - 40
                 def getData(self, seconds=30, nTries=5):
                     '''
                         This method gets the data from files and put the data into the dataOut object
                         In addition, increase el the buffer counter in one.
                         Return:
                             data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                         buffer. Si no hay mas archivos a leer retorna None.
                         Affected:
                             self.dataOut
                             self.profileIndex
                             self.flagDiscontinuousBlock
                             self.flagIsNewBlock
                     '''
                     #print("getdata")
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         #print("hi")
                         self.__flagDiscontinuousBlock = False
                         while True:
                             #print ("q ha pasado")
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond * self.__sample_rate:
                                 raise schainpy.admin.SchainError('Error')
                                 return
                             if self.__flagDiscontinuousBlock:
                                 raise schainpy.admin.SchainError('discontinuous block found')
                                 return
                             if not self.__online:
                                 raise schainpy.admin.SchainError('Online?')
                                 return
                             err_counter += 1
                             if err_counter > nTries:
                                 raise schainpy.admin.SchainError('Max retrys reach')
                                 return
                             print('[Reading] waiting %d seconds to read a new block' % seconds)
                             time.sleep(seconds)
                     self.dataOut.data                   = self.__data_buffer[:, self.__bufferIndex:self.__bufferIndex + self.__nSamples]
                     self.dataOut.utctime                = ( self.__thisUnixSample + self.__bufferIndex) / self.__sample_rate
                     self.dataOut.flagNoData             = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.dataOut.profileIndex           = self.profileIndex
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex  += 1
                     if self.profileIndex == self.dataOut.nProfiles:
                         self.profileIndex = 0
                     return True
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
                     # self.systemHeaderObj.printInfo()
                     # self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     return
                     # print self.profileIndex
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                     #self.i = self.i+1
                     self.getData(seconds=self.__delay)
                     return
             class DigitalRFWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     Operation.__init__(self, **kwargs)
                     self.metadata_dict = {}
                     self.dataOut       = None
                     self.dtype         = None
                     self.oldAverage    = 0
                 def setHeader(self):
                     self.metadata_dict['frequency']      = self.dataOut.frequency
                     self.metadata_dict['timezone']       = self.dataOut.timeZone
                     self.metadata_dict['dtype']          = pickle.dumps(self.dataOut.dtype)
                     self.metadata_dict['nProfiles']      = self.dataOut.nProfiles
                     self.metadata_dict['heightList']     = self.dataOut.heightList
                     self.metadata_dict['channelList']    = self.dataOut.channelList
                     self.metadata_dict['flagDecodeData'] = self.dataOut.flagDecodeData
                     self.metadata_dict['flagDeflipData'] = self.dataOut.flagDeflipData
                     self.metadata_dict['flagShiftFFT']   = self.dataOut.flagShiftFFT
                     self.metadata_dict['useLocalTime']   = self.dataOut.useLocalTime
                     self.metadata_dict['nCohInt']        = self.dataOut.nCohInt
                     self.metadata_dict['type']           = self.dataOut.type
                     self.metadata_dict['flagDataAsBlock']= getattr(
                         self.dataOut, 'flagDataAsBlock', None)  # chequear
                 def setup(self, dataOut, path, frequency, fileCadence, dirCadence, metadataCadence, set=0, metadataFile='metadata', ext='.h5'):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataOut: Input data will also be outputa data
                     '''
                     self.setHeader()
                     self.__ippSeconds  = dataOut.ippSeconds
                     self.__deltaH      = dataOut.getDeltaH()
                     self.__sample_rate = 1e6 * 0.15 / self.__deltaH
                     self.__dtype       = dataOut.dtype
                     if len(dataOut.dtype) == 2:
                         self.__dtype = dataOut.dtype[0]
                     self.__nSamples  = dataOut.systemHeaderObj.nSamples
                     self.__nProfiles = dataOut.nProfiles
                     if self.dataOut.type != 'Voltage':
                         raise 'Digital RF cannot be used with this data type'
                         self.arr_data = numpy.ones((1, dataOut.nFFTPoints * len(
                             self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
                     else:
                         self.arr_data = numpy.ones((self.__nSamples, len(
                             self.dataOut.channelList)), dtype=[('r', self.__dtype), ('i', self.__dtype)])
                     file_cadence_millisecs  = 1000
                     sample_rate_fraction    = Fraction(self.__sample_rate).limit_denominator()
                     sample_rate_numerator   = int(sample_rate_fraction.numerator)
                     sample_rate_denominator = int(sample_rate_fraction.denominator)
                     start_global_index      = dataOut.utctime * self.__sample_rate
                     uuid              = 'prueba'
                     compression_level = 0
                     checksum          = False
                     is_complex        = True
                     num_subchannels   = len(dataOut.channelList)
                     is_continuous     = True
                     marching_periods  = False
                     self.digitalWriteObj = digital_rf.DigitalRFWriter(path, self.__dtype, dirCadence,
                                                                       fileCadence, start_global_index,
                                                                       sample_rate_numerator, sample_rate_denominator, uuid, compression_level, checksum,
                                                                       is_complex, num_subchannels, is_continuous, marching_periods)
                     metadata_dir         = os.path.join(path, 'metadata')
                     os.system('mkdir %s' % (metadata_dir))
                     self.digitalMetadataWriteObj = digital_rf.DigitalMetadataWriter(metadata_dir, dirCadence, 1,  # 236, file_cadence_millisecs / 1000
                                                                                     sample_rate_numerator, sample_rate_denominator,
                                                                                     metadataFile)
                     self.isConfig      = True
                     self.currentSample = 0
                     self.oldAverage    = 0
                     self.count         = 0
                     return
                 def writeMetadata(self):
                     start_idx                                   = self.__sample_rate * self.dataOut.utctime
                     self.metadata_dict['processingHeader']      = self.dataOut.processingHeaderObj.getAsDict(
                     )
                     self.metadata_dict['radarControllerHeader'] = self.dataOut.radarControllerHeaderObj.getAsDict(
                     )
                     self.metadata_dict['systemHeader']          = self.dataOut.systemHeaderObj.getAsDict(
                     )
                     self.digitalMetadataWriteObj.write(start_idx, self.metadata_dict)
                     return
                 def timeit(self, toExecute):
                     t0 = time()
                     toExecute()
                     self.executionTime  = time() - t0
                     if self.oldAverage is None:
                         self.oldAverage = self.executionTime
                     self.oldAverage     = (self.executionTime + self.count *
                                        self.oldAverage) / (self.count + 1.0)
                     self.count          = self.count + 1.0
                     return
                 def writeData(self):
                     if self.dataOut.type != 'Voltage':
                         raise 'Digital RF cannot be used with this data type'
                         for channel in self.dataOut.channelList:
                             for i in range(self.dataOut.nFFTPoints):
                                 self.arr_data[1][channel * self.dataOut.nFFTPoints +
                                                  i]['r'] = self.dataOut.data[channel][i].real
                                 self.arr_data[1][channel * self.dataOut.nFFTPoints +
                                                  i]['i'] = self.dataOut.data[channel][i].imag
                     else:
                         for i in range(self.dataOut.systemHeaderObj.nSamples):
                             for channel in self.dataOut.channelList:
                                 self.arr_data[i][channel]['r'] = self.dataOut.data[channel][i].real
                                 self.arr_data[i][channel]['i'] = self.dataOut.data[channel][i].imag
                     def f(): return self.digitalWriteObj.rf_write(self.arr_data)
                     self.timeit(f)
                     return
                 def run(self, dataOut, frequency=49.92e6, path=None, fileCadence=1000, dirCadence=36000, metadataCadence=1, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataOut: object with the data
                     '''
                     # print dataOut.__dict__
                     self.dataOut = dataOut
                     if not self.isConfig:
                         self.setup(dataOut, path, frequency, fileCadence,
                                    dirCadence, metadataCadence, **kwargs)
                         self.writeMetadata()
                     self.writeData()
                     ## self.currentSample += 1
                     # if self.dataOut.flagDataAsBlock or self.currentSample == 1:
                     # self.writeMetadata()
                     ## if self.currentSample == self.__nProfiles: self.currentSample = 0
                     return dataOut# en la version 2.7 no aparece este return
                 def close(self):
                     print('[Writing] - Closing files ')
                     print('Average of writing to digital rf format is ', self.oldAverage * 1000)
                     try:
                         self.digitalWriteObj.close()
                     except:
                         pass

schainpy/model/io/jroIO_heispectra.py +1 -1

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import os, sys
             import time, datetime
             import numpy
             import fnmatch
             import glob
             from time import sleep
             try:
                 import pyfits
             except ImportError as e:
-                print("Fits data cannot be used. Install pyfits module")
+                pass
             from xml.etree.ElementTree import ElementTree
             from .jroIO_base import isRadarFolder, isNumber
             from schainpy.model.data.jrodata import Fits
             from schainpy.model.proc.jroproc_base import Operation, ProcessingUnit, MPDecorator
             from schainpy.utils import log
             class PyFits(object):
                 name=None
                 format=None
                 array =None
                 data =None
                 thdulist=None
                 prihdr=None
                 hdu=None
                 def __init__(self):
                     pass
                 def setColF(self,name,format,array):
                     self.name=name
                     self.format=format
                     self.array=array
                     a1=numpy.array([self.array],dtype=numpy.float32)
                     self.col1 = pyfits.Column(name=self.name, format=self.format, array=a1)
                     return self.col1
             #    def setColP(self,name,format,data):
             #        self.name=name
             #        self.format=format
             #        self.data=data
             #        a2=numpy.array([self.data],dtype=numpy.float32)
             #        self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
             #        return self.col2
                 def writeData(self,name,format,data):
                     self.name=name
                     self.format=format
                     self.data=data
                     a2=numpy.array([self.data],dtype=numpy.float32)
                     self.col2 = pyfits.Column(name=self.name, format=self.format, array=a2)
                     return self.col2
                 def cFImage(self,idblock,year,month,day,hour,minute,second):
                     self.hdu= pyfits.PrimaryHDU(idblock)
                     self.hdu.header.set("Year",year)
                     self.hdu.header.set("Month",month)
                     self.hdu.header.set("Day",day)
                     self.hdu.header.set("Hour",hour)
                     self.hdu.header.set("Minute",minute)
                     self.hdu.header.set("Second",second)
                     return self.hdu
                 def Ctable(self,colList):
                     self.cols=pyfits.ColDefs(colList)
                     self.tbhdu = pyfits.new_table(self.cols)
                     return self.tbhdu
                 def CFile(self,hdu,tbhdu):
                     self.thdulist=pyfits.HDUList([hdu,tbhdu])
                 def wFile(self,filename):
                     if os.path.isfile(filename):
                         os.remove(filename)
                     self.thdulist.writeto(filename)
             class ParameterConf:
                 ELEMENTNAME = 'Parameter'
                 def __init__(self):
                     self.name = ''
                     self.value = ''
                 def readXml(self, parmElement):
                     self.name = parmElement.get('name')
                     self.value = parmElement.get('value')
                 def getElementName(self):
                     return self.ELEMENTNAME
             class Metadata(object):
                 def __init__(self, filename):
                     self.parmConfObjList = []
                     self.readXml(filename)
                 def readXml(self, filename):
                     self.projectElement = None
                     self.procUnitConfObjDict = {}
                     self.projectElement = ElementTree().parse(filename)
                     self.project = self.projectElement.tag
                     parmElementList = self.projectElement.getiterator(ParameterConf().getElementName())
                     for parmElement in parmElementList:
                         parmConfObj = ParameterConf()
                         parmConfObj.readXml(parmElement)
                         self.parmConfObjList.append(parmConfObj)
             class FitsWriter(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.isConfig = False
                     self.dataBlocksPerFile = None
                     self.blockIndex = 0
                     self.flagIsNewFile = 1
                     self.fitsObj = None
                     self.optchar = 'P'
                     self.ext = '.fits'
                     self.setFile = 0
                 def setFitsHeader(self, dataOut, metadatafile=None):
                     header_data = pyfits.PrimaryHDU()
                     header_data.header['EXPNAME'] = "RADAR DATA"
                     header_data.header['DATATYPE'] = "SPECTRA"
                     header_data.header['COMMENT'] = ""
                     if metadatafile:
                         metadata4fits = Metadata(metadatafile)
                         for parameter in metadata4fits.parmConfObjList:
                             parm_name = parameter.name
                             parm_value = parameter.value
                             header_data.header[parm_name] = parm_value
                     header_data.header['DATETIME'] = time.strftime("%b %d %Y %H:%M:%S", dataOut.datatime.timetuple())
                     header_data.header['CHANNELLIST'] = str(dataOut.channelList)
                     header_data.header['NCHANNELS'] = dataOut.nChannels
                     #header_data.header['HEIGHTS'] = dataOut.heightList
                     header_data.header['NHEIGHTS'] = dataOut.nHeights
                     header_data.header['IPPSECONDS'] = dataOut.ippSeconds
                     header_data.header['NCOHINT'] = dataOut.nCohInt
                     header_data.header['NINCOHINT'] = dataOut.nIncohInt
                     header_data.header['TIMEZONE'] = dataOut.timeZone
                     header_data.header['NBLOCK'] = self.blockIndex
                     header_data.writeto(self.filename)
                     self.addExtension(dataOut.heightList,'HEIGHTLIST')
                 def setup(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None):
                     self.path = path
                     self.dataOut = dataOut
                     self.metadatafile = metadatafile
                     self.dataBlocksPerFile = dataBlocksPerFile
                 def open(self):
                     self.fitsObj = pyfits.open(self.filename, mode='update')
                 def addExtension(self, data, tagname):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=tagname)
                     #extension.header['TAG'] = tagname
                     self.fitsObj.append(extension)
                     self.write()
                 def addData(self, data):
                     self.open()
                     extension = pyfits.ImageHDU(data=data, name=self.fitsObj[0].header['DATATYPE'])
                     extension.header['UTCTIME'] = self.dataOut.utctime
                     self.fitsObj.append(extension)
                     self.blockIndex += 1
                     self.fitsObj[0].header['NBLOCK'] = self.blockIndex
                     self.write()
                 def write(self):
                     self.fitsObj.flush(verbose=True)
                     self.fitsObj.close()
                 def setNextFile(self):
                     ext = self.ext
                     path = self.path
                     timeTuple = time.localtime( self.dataOut.utctime)
                     subfolder = 'd%4.4d%3.3d' % (timeTuple.tm_year,timeTuple.tm_yday)
                     fullpath = os.path.join( path, subfolder )
                     if not( os.path.exists(fullpath) ):
                         os.mkdir(fullpath)
                         self.setFile = -1 #inicializo mi contador de seteo
                     else:
                         filesList = os.listdir( fullpath )
                         if len( filesList ) > 0:
                             filesList = sorted( filesList, key=str.lower )
                             filen = filesList[-1]
                             if isNumber( filen[8:11] ):
                                 self.setFile = int( filen[8:11] ) #inicializo mi contador de seteo al seteo del ultimo file
                             else:
                                 self.setFile = -1
                         else:
                             self.setFile = -1 #inicializo mi contador de seteo
                     setFile = self.setFile
                     setFile += 1
                     thisFile = '%s%4.4d%3.3d%3.3d%s' % (self.optchar,
                                                     timeTuple.tm_year,
                                                     timeTuple.tm_yday,
                                                     setFile,
                                                     ext )
                     filename = os.path.join( path, subfolder, thisFile )
                     self.blockIndex = 0
                     self.filename = filename
                     self.setFile = setFile
                     self.flagIsNewFile = 1
                     print('Writing the file: %s'%self.filename)
                     self.setFitsHeader(self.dataOut, self.metadatafile)
                     return 1
                 def writeBlock(self):
                     self.addData(self.dataOut.data_spc)
                     self.flagIsNewFile = 0
                 def __setNewBlock(self):
                     if self.flagIsNewFile:
                         return 1
                     if self.blockIndex < self.dataBlocksPerFile:
                         return 1
                     if not( self.setNextFile() ):
                         return 0
                     return 1
                 def writeNextBlock(self):
                     if not( self.__setNewBlock() ):
                         return 0
                     self.writeBlock()
                     return 1
                 def putData(self):
                     if self.flagIsNewFile:
                         self.setNextFile()
                     self.writeNextBlock()
                 def run(self, dataOut, path, dataBlocksPerFile=100, metadatafile=None, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, path, dataBlocksPerFile=dataBlocksPerFile, metadatafile=metadatafile, **kwargs)
                         self.isConfig = True
                     self.putData()
             @MPDecorator
             class FitsReader(ProcessingUnit):
             #     __TIMEZONE = time.timezone
                 expName = None
                 datetimestr = None
                 utc = None
                 nChannels = None
                 nSamples = None
                 dataBlocksPerFile = None
                 comments = None
                 lastUTTime = None
                 header_dict = None
                 data = None
                 data_header_dict = None
                 def __init__(self):#, **kwargs):
                     ProcessingUnit.__init__(self)#, **kwargs)
                     self.isConfig = False
                     self.ext = '.fits'
                     self.setFile = 0
                     self.flagNoMoreFiles = 0
                     self.flagIsNewFile = 1
                     self.flagDiscontinuousBlock = None
                     self.fileIndex = None
                     self.filename = None
                     self.fileSize = None
                     self.fitsObj = None
                     self.timeZone = None
                     self.nReadBlocks = 0
                     self.nTotalBlocks = 0
                     self.dataOut = self.createObjByDefault()
                     self.maxTimeStep = 10# deberia ser definido por el usuario usando el metodo setup()
                     self.blockIndex = 1
                 def createObjByDefault(self):
                     dataObj = Fits()
                     return dataObj
                 def isFileinThisTime(self, filename, startTime, endTime, useLocalTime=False):
                     try:
                         fitsObj = pyfits.open(filename,'readonly')
                     except:
                         print("File %s can't be opened" %(filename))
                         return None
                     header = fitsObj[0].header
                     struct_time = time.strptime(header['DATETIME'], "%b %d %Y %H:%M:%S")
                     utc = time.mktime(struct_time) - time.timezone #TIMEZONE debe ser un parametro del header FITS
                     ltc = utc
                     if useLocalTime:
                         ltc -= time.timezone
                     thisDatetime = datetime.datetime.utcfromtimestamp(ltc)
                     thisTime = thisDatetime.time()
                     if not ((startTime <= thisTime) and (endTime > thisTime)):
                         return None
                     return thisDatetime
                 def __setNextFileOnline(self):
                     raise NotImplementedError
                 def __setNextFileOffline(self):
                     idFile = self.fileIndex
                     while (True):
                         idFile += 1
                         if not(idFile < len(self.filenameList)):
                             self.flagNoMoreFiles = 1
                             print("No more Files")
                             return 0
                         filename = self.filenameList[idFile]
             #            if not(self.__verifyFile(filename)):
             #                continue
                         fileSize = os.path.getsize(filename)
                         fitsObj = pyfits.open(filename,'readonly')
                         break
                     self.flagIsNewFile = 1
                     self.fileIndex = idFile
                     self.filename = filename
                     self.fileSize = fileSize
                     self.fitsObj = fitsObj
                     self.blockIndex = 0
                     print("Setting the file: %s"%self.filename)
                     return 1
                 def __setValuesFromHeader(self):
                     self.dataOut.header = self.header_dict
                     self.dataOut.expName = self.expName
                     self.dataOut.timeZone = self.timeZone
                     self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
                     self.dataOut.comments = self.comments
             #         self.dataOut.timeInterval = self.timeInterval
                     self.dataOut.channelList = self.channelList
                     self.dataOut.heightList = self.heightList
                     self.dataOut.nCohInt = self.nCohInt
                     self.dataOut.nIncohInt = self.nIncohInt
                     self.dataOut.ipp_sec = self.ippSeconds
                 def readHeader(self):
                     headerObj = self.fitsObj[0]
                     self.header_dict = headerObj.header
                     if 'EXPNAME' in list(headerObj.header.keys()):
                         self.expName = headerObj.header['EXPNAME']
                     if 'DATATYPE' in list(headerObj.header.keys()):
                         self.dataType = headerObj.header['DATATYPE']
                     self.datetimestr = headerObj.header['DATETIME']
                     channelList = headerObj.header['CHANNELLIST']
                     channelList = channelList.split('[')
                     channelList = channelList[1].split(']')
                     channelList = channelList[0].split(',')
                     channelList = [int(ch) for ch in channelList]
                     self.channelList = channelList
                     self.nChannels = headerObj.header['NCHANNELS']
                     self.nHeights = headerObj.header['NHEIGHTS']
                     self.ippSeconds = headerObj.header['IPPSECONDS']
                     self.nCohInt = headerObj.header['NCOHINT']
                     self.nIncohInt = headerObj.header['NINCOHINT']
                     self.dataBlocksPerFile = headerObj.header['NBLOCK']
                     self.timeZone = headerObj.header['TIMEZONE']
             #         self.timeInterval = self.ippSeconds * self.nCohInt * self.nIncohInt
                     if 'COMMENT' in list(headerObj.header.keys()):
                         self.comments = headerObj.header['COMMENT']
                     self.readHeightList()
                 def readHeightList(self):
                     self.blockIndex = self.blockIndex + 1
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def readExtension(self):
                     obj = self.fitsObj[self.blockIndex]
                     self.heightList = obj.data
                     self.blockIndex = self.blockIndex + 1
                 def setNextFile(self):
                     if self.online:
                         newFile = self.__setNextFileOnline()
                     else:
                         newFile = self.__setNextFileOffline()
                     if not(newFile):
                         return 0
                     self.readHeader()
                     self.__setValuesFromHeader()
                     self.nReadBlocks = 0
             #         self.blockIndex = 1
                     return 1
                 def searchFilesOffLine(self,
                                         path,
                                         startDate,
                                         endDate,
                                         startTime=datetime.time(0,0,0),
                                         endTime=datetime.time(23,59,59),
                                         set=None,
                                         expLabel='',
                                         ext='.fits',
                                         walk=True):
                     pathList = []
                     if not walk:
                         pathList.append(path)
                     else:
                         dirList = []
                         for thisPath in os.listdir(path):
                             if not os.path.isdir(os.path.join(path,thisPath)):
                                 continue
                             if not isRadarFolder(thisPath):
                                 continue
                             dirList.append(thisPath)
                         if not(dirList):
                             return None, None
                         thisDate = startDate
                         while(thisDate <= endDate):
                             year = thisDate.timetuple().tm_year
                             doy = thisDate.timetuple().tm_yday
                             matchlist = fnmatch.filter(dirList, '?' + '%4.4d%3.3d' % (year,doy) + '*')
                             if len(matchlist) == 0:
                                 thisDate += datetime.timedelta(1)
                                 continue
                             for match in matchlist:
                                 pathList.append(os.path.join(path,match,expLabel))
                             thisDate += datetime.timedelta(1)
                     if pathList == []:
                         print("Any folder was found for the date range: %s-%s" %(startDate, endDate))
                         return None, None
                     print("%d folder(s) was(were) found for the date range: %s - %s" %(len(pathList), startDate, endDate))
                     filenameList = []
                     datetimeList = []
                     for i in range(len(pathList)):
                         thisPath = pathList[i]
                         fileList = glob.glob1(thisPath, "*%s" %ext)
                         fileList.sort()
                         for thisFile in fileList:
                             filename = os.path.join(thisPath,thisFile)
                             thisDatetime = self.isFileinThisTime(filename, startTime, endTime)
                             if not(thisDatetime):
                                 continue
                             filenameList.append(filename)
                             datetimeList.append(thisDatetime)
                     if not(filenameList):
                         print("Any file was found for the time range %s - %s" %(startTime, endTime))
                         return None, None
                     print("%d file(s) was(were) found for the time range: %s - %s" %(len(filenameList), startTime, endTime))
                     print()
                     for i in range(len(filenameList)):
                         print("%s -> [%s]" %(filenameList[i], datetimeList[i].ctime()))
                     self.filenameList = filenameList
                     self.datetimeList = datetimeList
                     return pathList, filenameList
                 def setup(self, path=None,
                             startDate=None,
                             endDate=None,
                             startTime=datetime.time(0,0,0),
                             endTime=datetime.time(23,59,59),
                             set=0,
                             expLabel = "",
                             ext = None,
                             online = False,
                             delay = 60,
                             walk = True):
                     if path == None:
                         raise ValueError("The path is not valid")
                     if ext == None:
                         ext = self.ext
                     if not(online):
                         print("Searching files in offline mode ...")
                         pathList, filenameList = self.searchFilesOffLine(path, startDate=startDate, endDate=endDate,
                                                                            startTime=startTime, endTime=endTime,
                                                                            set=set, expLabel=expLabel, ext=ext,
                                                                            walk=walk)
                         if not(pathList):
                             print("No *%s files into the folder %s \nfor the range: %s - %s"%(ext, path,
                                                                     datetime.datetime.combine(startDate,startTime).ctime(),
                                                                     datetime.datetime.combine(endDate,endTime).ctime()))
                             sys.exit(-1)
                         self.fileIndex = -1
                         self.pathList = pathList
                         self.filenameList = filenameList
                     self.online = online
                     self.delay = delay
                     ext = ext.lower()
                     self.ext = ext
                     if not(self.setNextFile()):
                         if (startDate!=None) and (endDate!=None):
                             print("No files in range: %s - %s" %(datetime.datetime.combine(startDate,startTime).ctime(), datetime.datetime.combine(endDate,endTime).ctime()))
                         elif startDate != None:
                             print("No files in range: %s" %(datetime.datetime.combine(startDate,startTime).ctime()))
                         else:
                             print("No files")
                         sys.exit(-1)
                 def readBlock(self):
                     dataObj = self.fitsObj[self.blockIndex]
                     self.data = dataObj.data
                     self.data_header_dict = dataObj.header
                     self.utc = self.data_header_dict['UTCTIME']
                     self.flagIsNewFile = 0
                     self.blockIndex += 1
                     self.nTotalBlocks += 1
                     self.nReadBlocks += 1
                     return 1
                 def __jumpToLastBlock(self):
                     raise NotImplementedError
                 def __waitNewBlock(self):
                     """
                     Return 1 si se encontro un nuevo bloque de datos, 0 de otra forma.
                     Si el modo de lectura es OffLine siempre retorn 0
                     """
                     if not self.online:
                         return 0
                     if (self.nReadBlocks >= self.dataBlocksPerFile):
                         return 0
                     currentPointer = self.fp.tell()
                     neededSize = self.processingHeaderObj.blockSize + self.basicHeaderSize
                     for nTries in range( self.nTries ):
                         self.fp.close()
                         self.fp = open( self.filename, 'rb' )
                         self.fp.seek( currentPointer )
                         self.fileSize = os.path.getsize( self.filename )
                         currentSize = self.fileSize - currentPointer
                         if ( currentSize >= neededSize ):
                             self.__rdBasicHeader()
                             return 1
                         print("\tWaiting %0.2f seconds for the next block, try %03d ..." % (self.delay, nTries+1))
                         sleep( self.delay )
                     return 0
                 def __setNewBlock(self):
                     if self.online:
                         self.__jumpToLastBlock()
                     if self.flagIsNewFile:
                         return 1
                     self.lastUTTime = self.utc
                     if self.online:
                         if self.__waitNewBlock():
                             return 1
                     if self.nReadBlocks < self.dataBlocksPerFile:
                         return 1
                     if not(self.setNextFile()):
                         return 0
                     deltaTime = self.utc - self.lastUTTime
                     self.flagDiscontinuousBlock = 0
                     if deltaTime > self.maxTimeStep:
                         self.flagDiscontinuousBlock = 1
                     return 1
                 def readNextBlock(self):
                     if not(self.__setNewBlock()):
                         return 0
                     if not(self.readBlock()):
                         return 0
                     return 1
                 def printInfo(self):
                     pass
                 def getData(self):
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         return (0, 'No more files')
                     self.flagDiscontinuousBlock = 0
                     self.flagIsNewBlock = 0
                     if not(self.readNextBlock()):
                         return (1, 'Error reading data')
                     if self.data is None:
                         self.dataOut.flagNoData = True
                         return (0, 'No more data')
                     self.dataOut.data = self.data
                     self.dataOut.data_header = self.data_header_dict
                     self.dataOut.utctime = self.utc
             #         self.dataOut.header = self.header_dict
             #         self.dataOut.expName = self.expName
             #         self.dataOut.nChannels = self.nChannels
             #         self.dataOut.timeZone = self.timeZone
             #         self.dataOut.dataBlocksPerFile = self.dataBlocksPerFile
             #         self.dataOut.comments = self.comments
             # #         self.dataOut.timeInterval = self.timeInterval
             #         self.dataOut.channelList = self.channelList
             #         self.dataOut.heightList = self.heightList
                     self.dataOut.flagNoData = False
                     # return self.dataOut.data
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
             @MPDecorator
             class SpectraHeisWriter(Operation):
             #    set = None
                 setFile = None
                 idblock = None
                 doypath = None
                 subfolder = None
                 def __init__(self):#, **kwargs):
                     Operation.__init__(self)#, **kwargs)
                     self.wrObj = PyFits()
             #        self.dataOut = dataOut
                     self.nTotalBlocks=0
             #        self.set = None
                     self.setFile = None
                     self.idblock = 0
                     self.wrpath = None
                     self.doypath = None
                     self.subfolder = None
                     self.isConfig = False
                 def isNumber(str):
                     """
                     Chequea si el conjunto de caracteres que componen un string puede ser convertidos a un numero.
                     Excepciones:
                     Si un determinado string no puede ser convertido a numero
                     Input:
                     str, string al cual se le analiza para determinar si convertible a un numero o no
                     Return:
                     True    :    si el string es uno numerico
                     False   :    no es un string numerico
                     """
                     try:
                         float( str )
                         return True
                     except:
                         return False
                 def setup(self, dataOut, wrpath):
                     if not(os.path.exists(wrpath)):
                         os.mkdir(wrpath)
                     self.wrpath = wrpath
             #        self.setFile = 0
                     self.dataOut = dataOut
                 def putData(self):
                     name= time.localtime( self.dataOut.utctime)
                     ext=".fits"
                     if self.doypath == None:
                        self.subfolder = 'F%4.4d%3.3d_%d' % (name.tm_year,name.tm_yday,time.mktime(datetime.datetime.now().timetuple()))
                        self.doypath = os.path.join( self.wrpath, self.subfolder )
                        os.mkdir(self.doypath)
                     if self.setFile == None:
             #           self.set = self.dataOut.set
                        self.setFile = 0
             #        if self.set != self.dataOut.set:
             ##            self.set = self.dataOut.set
             #            self.setFile = 0
                     #make the filename
                     thisFile = 'D%4.4d%3.3d_%3.3d%s' % (name.tm_year,name.tm_yday,self.setFile,ext)
                     filename = os.path.join(self.wrpath,self.subfolder, thisFile)
                     idblock = numpy.array([self.idblock],dtype="int64")
                     header=self.wrObj.cFImage(idblock=idblock,
                                             year=time.gmtime(self.dataOut.utctime).tm_year,
                                             month=time.gmtime(self.dataOut.utctime).tm_mon,
                                             day=time.gmtime(self.dataOut.utctime).tm_mday,
                                             hour=time.gmtime(self.dataOut.utctime).tm_hour,
                                             minute=time.gmtime(self.dataOut.utctime).tm_min,
                                             second=time.gmtime(self.dataOut.utctime).tm_sec)
                     c=3E8
                     deltaHeight = self.dataOut.heightList[1] - self.dataOut.heightList[0]
                     freq=numpy.arange(-1*self.dataOut.nHeights/2.,self.dataOut.nHeights/2.)*(c/(2*deltaHeight*1000))
                     colList = []
                     colFreq=self.wrObj.setColF(name="freq", format=str(self.dataOut.nFFTPoints)+'E', array=freq)
                     colList.append(colFreq)
                     nchannel=self.dataOut.nChannels
                     for i in range(nchannel):
                         col = self.wrObj.writeData(name="PCh"+str(i+1),
                                                      format=str(self.dataOut.nFFTPoints)+'E',
                                                       data=10*numpy.log10(self.dataOut.data_spc[i,:]))
                         colList.append(col)
                     data=self.wrObj.Ctable(colList=colList)
                     self.wrObj.CFile(header,data)
                     self.wrObj.wFile(filename)
                     #update the setFile
                     self.setFile += 1
                     self.idblock += 1
                     return 1
                 def run(self, dataOut, **kwargs):
                     if not(self.isConfig):
                         self.setup(dataOut, **kwargs)
                         self.isConfig = True
                     self.putData()
                     return dataOut

schainpy/model/io/jroIO_madrigal.py +1 -3

             '''
             Created on Aug 1, 2017
             @author: Juan C. Espinoza
             '''
             import os
             import sys
             import time
             import json
             import glob
             import datetime
             import numpy
             import h5py
             import schainpy.admin
             from schainpy.model.io.jroIO_base import LOCALTIME, Reader
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Parameters
             from schainpy.utils import log
             try:
                 import madrigal.cedar
             except:
-                log.warning(
+                pass
-                    'You should install "madrigal library" module if you want to read/write Madrigal data'
             try:
                 basestring
             except:
                 basestring = str
             DEF_CATALOG = {
                 'principleInvestigator': 'Marco Milla',
                 'expPurpose': '',
                 'cycleTime': '',
                 'correlativeExp': '',
                 'sciRemarks': '',
                 'instRemarks': ''
                 }
             DEF_HEADER = {
                 'kindatDesc': '',
                 'analyst': 'Jicamarca User',
                 'comments': '',
                 'history': ''
                 }
             MNEMONICS = {
 : 'jro',
 : 'jbr',
 : 'jul',
 : 'jas',
 : 'pbr',
 : 'hbr',
 : 'obr',
 : 'clr'
             }
             UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
             def load_json(obj):
                 '''
                 Parse json as string instead of unicode
                 '''
                 if isinstance(obj, str):
                     iterable = json.loads(obj)
                 else:
                     iterable = obj
                 if isinstance(iterable, dict):
                     return {str(k): load_json(v) if isinstance(v, dict) else str(v) if isinstance(v, basestring) else v
                         for k, v in list(iterable.items())}
                 elif isinstance(iterable, (list, tuple)):
                     return [str(v) if isinstance(v, basestring) else v for v in iterable]
                 return iterable
             @MPDecorator
             class MADReader(Reader, ProcessingUnit):
                 def __init__(self):
                     ProcessingUnit.__init__(self)
                     self.dataOut = Parameters()
                     self.counter_records = 0
                     self.nrecords = None
                     self.flagNoMoreFiles = 0
                     self.filename = None
                     self.intervals = set()
                     self.datatime = datetime.datetime(1900,1,1)
                     self.format = None
                     self.filefmt = "***%Y%m%d*******"
                 def setup(self, **kwargs):
                     self.set_kwargs(**kwargs)
                     self.oneDDict = load_json(self.oneDDict)
                     self.twoDDict = load_json(self.twoDDict)
                     self.ind2DList = load_json(self.ind2DList)
                     self.independentParam = self.ind2DList[0]
                     if self.path is None:
                         raise ValueError('The path is not valid')
                     self.open_file = open
                     self.open_mode = 'rb'
                     if self.format is None:
                         raise ValueError('The format is not valid choose simple or hdf5')
                     elif self.format.lower() in ('simple', 'txt'):
                         self.ext = '.txt'
                     elif self.format.lower() in ('cedar',):
                         self.ext = '.001'
                     else:
                         self.ext = '.hdf5'
                         self.open_file = h5py.File
                         self.open_mode = 'r'
                     if self.online:
                         log.log("Searching files in online mode...", self.name)
                         for nTries in range(self.nTries):
                             fullpath = self.searchFilesOnLine(self.path, self.startDate,
                                 self.endDate, self.expLabel, self.ext, self.walk,
                                 self.filefmt, self.folderfmt)
                             try:
                                 fullpath = next(fullpath)
                             except:
                                 fullpath = None
                             if fullpath:
                                 break
                             log.warning(
                                 'Waiting {} sec for a valid file in {}: try {} ...'.format(
                                     self.delay, self.path, nTries + 1),
                                 self.name)
                             time.sleep(self.delay)
                         if not(fullpath):
                             raise schainpy.admin.SchainError(
                                 'There isn\'t any valid file in {}'.format(self.path))
                     else:
                         log.log("Searching files in {}".format(self.path), self.name)
                         self.filenameList = self.searchFilesOffLine(self.path, self.startDate,
                             self.endDate, self.expLabel, self.ext, self.walk, self.filefmt, self.folderfmt)
                     self.setNextFile()
                 def readFirstHeader(self):
                     '''Read header and data'''
                     self.parseHeader()
                     self.parseData()
                     self.blockIndex = 0
                     return
                 def parseHeader(self):
                     '''
                     '''
                     self.output = {}
                     self.version = '2'
                     s_parameters = None
                     if self.ext == '.txt':
                         self.parameters = [s.strip().lower() for s in self.fp.readline().decode().strip().split(' ') if s]
                     elif self.ext == '.hdf5':
                         self.metadata = self.fp['Metadata']
                         if '_record_layout' in self.metadata:
                             s_parameters = [s[0].lower().decode() for s in self.metadata['Independent Spatial Parameters']]
                             self.version = '3'
                         self.parameters = [s[0].lower().decode() for s in self.metadata['Data Parameters']]
                     log.success('Parameters found: {}'.format(self.parameters),
                                 'MADReader')
                     if s_parameters:
                         log.success('Spatial parameters found: {}'.format(s_parameters),
                                     'MADReader')
                     for param in list(self.oneDDict.keys()):
                         if param.lower() not in self.parameters:
                             log.warning(
                                 'Parameter {} not found will be ignored'.format(
                                     param),
                                 'MADReader')
                             self.oneDDict.pop(param, None)
                     for param, value in list(self.twoDDict.items()):
                         if param.lower() not in self.parameters:
                             log.warning(
                                 'Parameter {} not found, it will be ignored'.format(
                                     param),
                                 'MADReader')
                             self.twoDDict.pop(param, None)
                             continue
                         if isinstance(value, list):
                             if value[0] not in self.output:
                                 self.output[value[0]] = []
                             self.output[value[0]].append([])
                 def parseData(self):
                     '''
                     '''
                     if self.ext == '.txt':
                         self.data = numpy.genfromtxt(self.fp, missing_values=('missing'))
                         self.nrecords = self.data.shape[0]
                         self.ranges = numpy.unique(self.data[:,self.parameters.index(self.independentParam.lower())])
                         self.counter_records = 0
                     elif self.ext == '.hdf5':
                         self.data = self.fp['Data']
                         self.ranges = numpy.unique(self.data['Table Layout'][self.independentParam.lower()])
                         self.times = numpy.unique(self.data['Table Layout']['ut1_unix'])
                         self.counter_records = int(self.data['Table Layout']['recno'][0])
                         self.nrecords = int(self.data['Table Layout']['recno'][-1])
                 def readNextBlock(self):
                     while True:
                         self.flagDiscontinuousBlock = 0
                         if self.counter_records == self.nrecords:
                             self.setNextFile()
                         self.readBlock()
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             log.warning(
                                 'Reading Record No. {}/{} -> {} [Skipping]'.format(
                                     self.counter_records,
                                     self.nrecords,
                                     self.datatime.ctime()),
                                 'MADReader')
                             continue
                         break
                     log.log(
                         'Reading Record No. {}/{} -> {}'.format(
                             self.counter_records,
                             self.nrecords,
                             self.datatime.ctime()),
                         'MADReader')
                     return 1
                 def readBlock(self):
                     '''
                     '''
                     dum = []
                     if self.ext == '.txt':
                         dt = self.data[self.counter_records][:6].astype(int)
                         if datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5]).date() > self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         self.datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                         while True:
                             dt = self.data[self.counter_records][:6].astype(int)
                             datatime = datetime.datetime(dt[0], dt[1], dt[2], dt[3], dt[4], dt[5])
                             if datatime == self.datatime:
                                 dum.append(self.data[self.counter_records])
                                 self.counter_records += 1
                                 if self.counter_records == self.nrecords:
                                     break
                                 continue
                             self.intervals.add((datatime-self.datatime).seconds)
                             break
                     elif self.ext == '.hdf5':
                         datatime = datetime.datetime.utcfromtimestamp(
                             self.times[self.counter_records])
                         dum = self.data['Table Layout'][self.data['Table Layout']['recno']==self.counter_records]
                         self.intervals.add((datatime-self.datatime).seconds)
                         if datatime.date()>self.datatime.date():
                             self.flagDiscontinuousBlock = 1
                         self.datatime = datatime
                         self.counter_records += 1
                     self.buffer = numpy.array(dum)
                     return
                 def set_output(self):
                     '''
                     Storing data from buffer to dataOut object
                     '''
                     parameters = [None for __ in self.parameters]
                     for param, attr in list(self.oneDDict.items()):
                         x = self.parameters.index(param.lower())
                         setattr(self.dataOut, attr, self.buffer[0][x])
                     for param, value in list(self.twoDDict.items()):
                         dummy = numpy.zeros(self.ranges.shape) + numpy.nan
                         if self.ext == '.txt':
                             x = self.parameters.index(param.lower())
                             y = self.parameters.index(self.independentParam.lower())
                             ranges = self.buffer[:,y]
                             #if self.ranges.size == ranges.size:
                             #    continue
                             index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
                             dummy[index] = self.buffer[:,x]
                         else:
                             ranges = self.buffer[self.independentParam.lower()]
                             index = numpy.where(numpy.in1d(self.ranges, ranges))[0]
                             dummy[index] = self.buffer[param.lower()]
                         if isinstance(value, str):
                             if value not in self.independentParam:
                                 setattr(self.dataOut, value, dummy.reshape(1,-1))
                         elif isinstance(value, list):
                             self.output[value[0]][value[1]] = dummy
                             parameters[value[1]] = param
                     for key, value in list(self.output.items()):
                         setattr(self.dataOut, key, numpy.array(value))
                     self.dataOut.parameters = [s for s in parameters if s]
                     self.dataOut.heightList = self.ranges
                     self.dataOut.utctime = (self.datatime - datetime.datetime(1970, 1, 1)).total_seconds()
                     self.dataOut.utctimeInit = self.dataOut.utctime
                     self.dataOut.paramInterval = min(self.intervals)
                     self.dataOut.useLocalTime = False
                     self.dataOut.flagNoData = False
                     self.dataOut.nrecords = self.nrecords
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                 def getData(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     if not  self.readNextBlock():
                         self.dataOut.flagNoData = True
                         return 0
                     self.set_output()
                     return 1
                 def run(self, **kwargs):
                     if not(self.isConfig):
                         self.setup(**kwargs)
                         self.isConfig = True
                     self.getData()
                     return
             @MPDecorator
             class MADWriter(Operation):
                 '''Writing module for Madrigal files
             type: external
             Inputs:
                         path        path where files will be created
                         oneDDict    json of one-dimensional parameters in record where keys
                                     are Madrigal codes (integers or mnemonics) and values the corresponding
                                     dataOut attribute e.g: {
                                         'gdlatr': 'lat',
                                         'gdlonr': 'lon',
                                         'gdlat2':'lat',
                                         'glon2':'lon'}
                         ind2DList   list of independent spatial two-dimensional parameters e.g:
                                     ['heigthList']
                         twoDDict    json of two-dimensional parameters in record where keys
                                     are Madrigal codes (integers or mnemonics) and values the corresponding
                                     dataOut attribute if multidimensional array specify as tupple
                                     ('attr', pos) e.g: {
                                         'gdalt': 'heightList',
                                         'vn1p2': ('data_output', 0),
                                         'vn2p2': ('data_output', 1),
                                         'vn3': ('data_output', 2),
                                         'snl': ('data_SNR', 'db')
                                         }
                         metadata    json of madrigal metadata (kinst, kindat, catalog and header)
                         format      hdf5, cedar
                         blocks      number of blocks per file'''
                 __attrs__ = ['path', 'oneDDict', 'ind2DList', 'twoDDict','metadata', 'format', 'blocks']
                 missing = -32767
                 def __init__(self):
                     Operation.__init__(self)
                     self.dataOut = Parameters()
                     self.counter = 0
                     self.path = None
                     self.fp = None
                 def run(self, dataOut, path, oneDDict, ind2DList='[]', twoDDict='{}',
                         metadata='{}', format='cedar', **kwargs):
                     if not self.isConfig:
                         self.setup(path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs)
                         self.isConfig = True
                     self.dataOut = dataOut
                     self.putData()
                     return 1
                 def setup(self, path, oneDDict, ind2DList, twoDDict, metadata, format, **kwargs):
                     '''
                     Configure Operation
                     '''
                     self.path = path
                     self.blocks = kwargs.get('blocks', None)
                     self.counter = 0
                     self.oneDDict = load_json(oneDDict)
                     self.twoDDict = load_json(twoDDict)
                     self.ind2DList = load_json(ind2DList)
                     meta = load_json(metadata)
                     self.kinst = meta.get('kinst')
                     self.kindat = meta.get('kindat')
                     self.catalog = meta.get('catalog', DEF_CATALOG)
                     self.header = meta.get('header', DEF_HEADER)
                     if format == 'cedar':
                         self.ext = '.dat'
                         self.extra_args = {}
                     elif format == 'hdf5':
                         self.ext = '.hdf5'
                         self.extra_args = {'ind2DList': self.ind2DList}
                     self.keys = [k.lower() for k in self.twoDDict]
                     if 'range' in self.keys:
                         self.keys.remove('range')
                     if 'gdalt' in self.keys:
                         self.keys.remove('gdalt')
                 def setFile(self):
                     '''
                     Create new cedar file object
                     '''
                     self.mnemonic = MNEMONICS[self.kinst]   #TODO get mnemonic from madrigal
                     date = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
                     filename = '{}{}{}'.format(self.mnemonic,
                                                date.strftime('%Y%m%d_%H%M%S'),
                                                self.ext)
                     self.fullname = os.path.join(self.path, filename)
                     if os.path.isfile(self.fullname) :
                         log.warning(
                             'Destination file {} already exists, previous file deleted.'.format(
                                 self.fullname),
                             'MADWriter')
                         os.remove(self.fullname)
                     try:
                         log.success(
                             'Creating file: {}'.format(self.fullname),
                             'MADWriter')
                         if not os.path.exists(self.path):
                             os.makedirs(self.path)
                         self.fp = madrigal.cedar.MadrigalCedarFile(self.fullname, True)
                     except ValueError as e:
                         log.error(
                             'Impossible to create a cedar object with "madrigal.cedar.MadrigalCedarFile"',
                             'MADWriter')
                         return
                     return 1
                 def writeBlock(self):
                     '''
                     Add data records to cedar file taking data from oneDDict and twoDDict
                     attributes.
                     Allowed parameters in: parcodes.tab
                     '''
                     startTime = datetime.datetime.utcfromtimestamp(self.dataOut.utctime)
                     endTime = startTime + datetime.timedelta(seconds=self.dataOut.paramInterval)
                     heights = self.dataOut.heightList
                     if self.ext == '.dat':
                         for key, value in list(self.twoDDict.items()):
                             if isinstance(value, str):
                                 data = getattr(self.dataOut, value)
                                 invalid = numpy.isnan(data)
                                 data[invalid] = self.missing
                             elif isinstance(value, (tuple, list)):
                                 attr, key = value
                                 data = getattr(self.dataOut, attr)
                                 invalid = numpy.isnan(data)
                                 data[invalid] = self.missing
                     out = {}
                     for key, value in list(self.twoDDict.items()):
                         key = key.lower()
                         if isinstance(value, str):
                             if 'db' in value.lower():
                                 tmp = getattr(self.dataOut, value.replace('_db', ''))
                                 SNRavg = numpy.average(tmp, axis=0)
                                 tmp = 10*numpy.log10(SNRavg)
                             else:
                                 tmp = getattr(self.dataOut, value)
                             out[key] = tmp.flatten()[:len(heights)]
                         elif isinstance(value, (tuple, list)):
                             attr, x = value
                             data = getattr(self.dataOut, attr)
                             out[key] = data[int(x)][:len(heights)]
                     a = numpy.array([out[k] for k in self.keys])
                     nrows = numpy.array([numpy.isnan(a[:, x]).all() for x in range(len(heights))])
                     index = numpy.where(nrows == False)[0]
                     rec = madrigal.cedar.MadrigalDataRecord(
                         self.kinst,
                         self.kindat,
                         startTime.year,
                         startTime.month,
                         startTime.day,
                         startTime.hour,
                         startTime.minute,
                         startTime.second,
                         startTime.microsecond/10000,
                         endTime.year,
                         endTime.month,
                         endTime.day,
                         endTime.hour,
                         endTime.minute,
                         endTime.second,
                         endTime.microsecond/10000,
                         list(self.oneDDict.keys()),
                         list(self.twoDDict.keys()),
                         len(index),
                         **self.extra_args
                     )
                     # Setting 1d values
                     for key in self.oneDDict:
                         rec.set1D(key, getattr(self.dataOut, self.oneDDict[key]))
                     # Setting 2d values
                     nrec = 0
                     for n in index:
                         for key in out:
                             rec.set2D(key, nrec, out[key][n])
                         nrec += 1
                     self.fp.append(rec)
                     if self.ext == '.hdf5' and self.counter % 500 == 0 and self.counter > 0:
                         self.fp.dump()
                     if self.counter % 20 == 0 and self.counter > 0:
                         log.log(
                             'Writing {} records'.format(
                                 self.counter),
                             'MADWriter')
                 def setHeader(self):
                     '''
                     Create an add catalog and header to cedar file
                     '''
                     log.success('Closing file {}'.format(self.fullname), 'MADWriter')
                     if self.ext == '.dat':
                         self.fp.write()
                     else:
                         self.fp.dump()
                         self.fp.close()
                     header = madrigal.cedar.CatalogHeaderCreator(self.fullname)
                     header.createCatalog(**self.catalog)
                     header.createHeader(**self.header)
                     header.write()
                 def putData(self):
                     if self.dataOut.flagNoData:
                         return 0
                     if self.dataOut.flagDiscontinuousBlock or self.counter == self.blocks:
                         if self.counter > 0:
                             self.setHeader()
                         self.counter = 0
                     if self.counter == 0:
                         self.setFile()
                     self.writeBlock()
                     self.counter += 1
                 def close(self):
                     if self.counter > 0:
                         self.setHeader()

schainpy/model/io/jroIO_usrp.py +1 -1

             '''
             Created on Jul 3, 2014
             @author: roj-idl71
             '''
             import os
             import datetime
             import numpy
             try:
                 from gevent import sleep
             except:
                 from time import sleep
             from schainpy.model.data.jroheaderIO import RadarControllerHeader, SystemHeader
             from schainpy.model.data.jrodata import Voltage
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             try:
                 import digital_rf_hdf5
             except:
-                print('You should install "digital_rf_hdf5" module if you want to read USRP data')
+                pass
             class USRPReader(ProcessingUnit):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     ProcessingUnit.__init__(self, **kwargs)
                     self.dataOut = Voltage()
                     self.__printInfo = True
                     self.__flagDiscontinuousBlock = False
                     self.__bufferIndex = 9999999
                     self.__ippKm = None
                     self.__codeType = 0
                     self.__nCode = None
                     self.__nBaud = None
                     self.__code = None
                 def __getCurrentSecond(self):
                     return self.__thisUnixSample/self.__sample_rate
                 thisSecond = property(__getCurrentSecond, "I'm the 'thisSecond' property.")
                 def __setFileHeader(self):
                     '''
                     In this method will be initialized every parameter of dataOut object (header, no data)
                     '''
                     ippSeconds = 1.0*self.__nSamples/self.__sample_rate
                     nProfiles = 1.0/ippSeconds  #Number of profiles in one second
                     self.dataOut.radarControllerHeaderObj = RadarControllerHeader(ippKm=self.__ippKm,
                                                                                   txA=0,
                                                                                   txB=0,
                                                                                   nWindows=1,
                                                                                   nHeights=self.__nSamples,
                                                                                   firstHeight=self.__firstHeigth,
                                                                                   deltaHeight=self.__deltaHeigth,
                                                                                   codeType=self.__codeType,
                                                                                   nCode=self.__nCode, nBaud=self.__nBaud,
                                                                                   code = self.__code)
                     self.dataOut.systemHeaderObj = SystemHeader(nSamples=self.__nSamples,
                                                                 nProfiles=nProfiles,
                                                                 nChannels=len(self.__channelList),
                                                                 adcResolution=14)
                     self.dataOut.type = "Voltage"
                     self.dataOut.data = None
                     self.dataOut.dtype = numpy.dtype([('real','<i8'),('imag','<i8')])
             #        self.dataOut.nChannels = 0
             #        self.dataOut.nHeights = 0
                     self.dataOut.nProfiles = nProfiles
                     self.dataOut.heightList = self.__firstHeigth + numpy.arange(self.__nSamples, dtype = numpy.float)*self.__deltaHeigth
                     self.dataOut.channelList = self.__channelList
                     self.dataOut.blocksize = self.dataOut.getNChannels() * self.dataOut.getNHeights()
             #        self.dataOut.channelIndexList = None
                     self.dataOut.flagNoData = True
                     #Set to TRUE if the data is discontinuous
                     self.dataOut.flagDiscontinuousBlock = False
                     self.dataOut.utctime = None
                     self.dataOut.timeZone = self.__timezone/60  #timezone like jroheader, difference in minutes between UTC and localtime
                     self.dataOut.dstFlag = 0
                     self.dataOut.errorCount = 0
                     self.dataOut.nCohInt = 1
                     self.dataOut.flagDecodeData = False #asumo que la data esta decodificada
                     self.dataOut.flagDeflipData = False #asumo que la data esta sin flip
                     self.dataOut.flagShiftFFT = False
                     self.dataOut.ippSeconds = ippSeconds
                     #Time interval between profiles
                     #self.dataOut.timeInterval = self.dataOut.ippSeconds * self.dataOut.nCohInt
                     self.dataOut.frequency = self.__frequency
                     self.dataOut.realtime = self.__online
                 def findDatafiles(self, path, startDate=None, endDate=None):
                     if not os.path.isdir(path):
                         return []
                     try:
                         digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = digitalReadObj.get_channels()
                     if not channelNameList:
                         return []
                     metadata_dict = digitalReadObj.get_rf_file_metadata(channelNameList[0])
                     sample_rate = metadata_dict['sample_rate'][0]
                     this_metadata_file = digitalReadObj.get_metadata(channelNameList[0])
                     try:
                         timezone = this_metadata_file['timezone'].value
                     except:
                         timezone = 0
                     startUTCSecond, endUTCSecond = digitalReadObj.get_bounds(channelNameList[0])/sample_rate - timezone
                     startDatetime = datetime.datetime.utcfromtimestamp(startUTCSecond)
                     endDatatime = datetime.datetime.utcfromtimestamp(endUTCSecond)
                     if not startDate:
                         startDate = startDatetime.date()
                     if not endDate:
                         endDate = endDatatime.date()
                     dateList = []
                     thisDatetime = startDatetime
                     while(thisDatetime<=endDatatime):
                         thisDate = thisDatetime.date()
                         if thisDate < startDate:
                             continue
                         if thisDate > endDate:
                             break
                         dateList.append(thisDate)
                         thisDatetime += datetime.timedelta(1)
                     return dateList
                 def setup(self, path = None,
                                 startDate = None,
                                 endDate = None,
                                 startTime = datetime.time(0,0,0),
                                 endTime = datetime.time(23,59,59),
                                 channelList = None,
                                 nSamples = None,
                                 ippKm = 60,
                                 online = False,
                                 delay = 60,
                                 buffer_size = 1024,
                                 **kwargs):
                     '''
                     In this method we should set all initial parameters.
                     Inputs:
                         path
                         startDate
                         endDate
                         startTime
                         endTime
                         set
                         expLabel
                         ext
                         online
                         delay
                     '''
                     if not os.path.isdir(path):
                         raise ValueError("[Reading] Directory %s does not exist" %path)
                     try:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path, load_all_metadata=True)
                     except:
                         self.digitalReadObj = digital_rf_hdf5.read_hdf5(path)
                     channelNameList = self.digitalReadObj.get_channels()
                     if not channelNameList:
                         raise ValueError("[Reading] Directory %s does not have any files" %path)
                     if not channelList:
                         channelList = list(range(len(channelNameList)))
                     ##########  Reading metadata ######################
                     metadata_dict = self.digitalReadObj.get_rf_file_metadata(channelNameList[channelList[0]])
                     self.__sample_rate = metadata_dict['sample_rate'][0]
             #         self.__samples_per_file = metadata_dict['samples_per_file'][0]
                     self.__deltaHeigth = 1e6*0.15/self.__sample_rate
                     this_metadata_file = self.digitalReadObj.get_metadata(channelNameList[channelList[0]])
                     self.__frequency = None
                     try:
                         self.__frequency = this_metadata_file['center_frequencies'].value
                     except:
                         self.__frequency = this_metadata_file['fc'].value
                     if not self.__frequency:
                         raise ValueError("Center Frequency is not defined in metadata file")
                     try:
                         self.__timezone = this_metadata_file['timezone'].value
                     except:
                         self.__timezone = 0
                     self.__firstHeigth = 0
                     try:
                         codeType = this_metadata_file['codeType'].value
                     except:
                         codeType = 0
                     nCode = 1
                     nBaud = 1
                     code = numpy.ones((nCode, nBaud), dtype=numpy.int)
                     if codeType:
                         nCode = this_metadata_file['nCode'].value
                         nBaud = this_metadata_file['nBaud'].value
                         code = this_metadata_file['code'].value
                     if not ippKm:
                         try:
                             #seconds to km
                             ippKm = 1e6*0.15*this_metadata_file['ipp'].value
                         except:
                             ippKm = None
                     ####################################################
                     startUTCSecond = None
                     endUTCSecond = None
                     if startDate:
                         startDatetime = datetime.datetime.combine(startDate, startTime)
                         startUTCSecond = (startDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
                     if endDate:
                         endDatetime = datetime.datetime.combine(endDate, endTime)
                         endUTCSecond = (endDatetime-datetime.datetime(1970,1,1)).total_seconds() + self.__timezone
                     start_index, end_index = self.digitalReadObj.get_bounds(channelNameList[channelList[0]])
                     if not startUTCSecond:
                         startUTCSecond = start_index/self.__sample_rate
                     if start_index > startUTCSecond*self.__sample_rate:
                         startUTCSecond = start_index/self.__sample_rate
                     if not endUTCSecond:
                         endUTCSecond = end_index/self.__sample_rate
                     if end_index < endUTCSecond*self.__sample_rate:
                         endUTCSecond = end_index/self.__sample_rate
                     if not nSamples:
                         if not ippKm:
                             raise ValueError("[Reading] nSamples or ippKm should be defined")
                         nSamples = int(ippKm / (1e6*0.15/self.__sample_rate))
                     channelBoundList = []
                     channelNameListFiltered = []
                     for thisIndexChannel in channelList:
                         thisChannelName =  channelNameList[thisIndexChannel]
                         start_index, end_index = self.digitalReadObj.get_bounds(thisChannelName)
                         channelBoundList.append((start_index, end_index))
                         channelNameListFiltered.append(thisChannelName)
                     self.profileIndex = 0
                     self.__delay = delay
                     self.__ippKm = ippKm
                     self.__codeType = codeType
                     self.__nCode = nCode
                     self.__nBaud = nBaud
                     self.__code = code
                     self.__datapath = path
                     self.__online = online
                     self.__channelList = channelList
                     self.__channelNameList = channelNameListFiltered
                     self.__channelBoundList = channelBoundList
                     self.__nSamples = nSamples
                     self.__samples_to_read = int(buffer_size*nSamples)
                     self.__nChannels = len(self.__channelList)
                     self.__startUTCSecond = startUTCSecond
                     self.__endUTCSecond = endUTCSecond
                     self.__timeInterval = 1.0 * self.__samples_to_read/self.__sample_rate #Time interval
                     if online:
             #             self.__thisUnixSample = int(endUTCSecond*self.__sample_rate - 4*self.__samples_to_read)
                         startUTCSecond = numpy.floor(endUTCSecond)
                     self.__thisUnixSample = int(startUTCSecond*self.__sample_rate) - self.__samples_to_read
                     self.__data_buffer = numpy.zeros((self.__nChannels, self.__samples_to_read), dtype = numpy.complex)
                     self.__setFileHeader()
                     self.isConfig = True
                     print("[Reading] USRP Data was found from %s to %s " %(
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   ))
                     print("[Reading] Starting process from %s to %s" %(datetime.datetime.utcfromtimestamp(startUTCSecond - self.__timezone),
                                                                        datetime.datetime.utcfromtimestamp(endUTCSecond - self.__timezone)
                                                                        ))
                 def __reload(self):
                     if not self.__online:
                         return
             #         print
             #         print "%s not in range [%s, %s]" %(
             #                                           datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
             #                                           datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
             #                                           )
                     print("[Reading] reloading metadata ...")
                     try:
                         self.digitalReadObj.reload(complete_update=True)
                     except:
                         self.digitalReadObj.reload()
                     start_index, end_index = self.digitalReadObj.get_bounds(self.__channelNameList[self.__channelList[0]])
                     if start_index > self.__startUTCSecond*self.__sample_rate:
                         self.__startUTCSecond = 1.0*start_index/self.__sample_rate
                     if end_index > self.__endUTCSecond*self.__sample_rate:
                         self.__endUTCSecond = 1.0*end_index/self.__sample_rate
                         print()
                         print("[Reading] New timerange found [%s, %s] " %(
                                                                   datetime.datetime.utcfromtimestamp(self.__startUTCSecond - self.__timezone),
                                                                   datetime.datetime.utcfromtimestamp(self.__endUTCSecond - self.__timezone)
                                                                   ))
                         return True
                     return False
                 def __readNextBlock(self, seconds=30, volt_scale = 218776):
                     '''
                     '''
                     #Set the next data
                     self.__flagDiscontinuousBlock = False
                     self.__thisUnixSample += self.__samples_to_read
                     if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
                         print("[Reading] There are no more data into selected time-range")
                         self.__reload()
                         if self.__thisUnixSample + 2*self.__samples_to_read > self.__endUTCSecond*self.__sample_rate:
                             self.__thisUnixSample -=  self.__samples_to_read
                             return False
                     indexChannel = 0
                     dataOk = False
                     for thisChannelName in self.__channelNameList:
                         try:
                             result = self.digitalReadObj.read_vector_c81d(self.__thisUnixSample,
                                                                           self.__samples_to_read,
                                                                           thisChannelName)
                         except IOError as e:
                             #read next profile
                             self.__flagDiscontinuousBlock = True
                             print("[Reading] %s" %datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone), e)
                             break
                         if result.shape[0] != self.__samples_to_read:
                             self.__flagDiscontinuousBlock = True
                             print("[Reading] %s: Too few samples were found, just %d/%d  samples" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                                             result.shape[0],
                                                                                             self.__samples_to_read))
                             break
                         self.__data_buffer[indexChannel,:] = result*volt_scale
                         indexChannel += 1
                         dataOk = True
                     self.__utctime = self.__thisUnixSample/self.__sample_rate
                     if not dataOk:
                         return False
                     print("[Reading] %s: %d samples <> %f sec" %(datetime.datetime.utcfromtimestamp(self.thisSecond - self.__timezone),
                                                                      self.__samples_to_read,
                                                                      self.__timeInterval))
                     self.__bufferIndex = 0
                     return True
                 def __isBufferEmpty(self):
                     if self.__bufferIndex <= self.__samples_to_read - self.__nSamples:
                         return False
                     return True
                 def getData(self, seconds=30, nTries=5):
                     '''
                     This method gets the data from files and put the data into the dataOut object
                     In addition, increase el the buffer counter in one.
                     Return:
                         data    :    retorna un perfil de voltages (alturas * canales) copiados desde el
                                      buffer. Si no hay mas archivos a leer retorna None.
                     Affected:
                         self.dataOut
                         self.profileIndex
                         self.flagDiscontinuousBlock
                         self.flagIsNewBlock
                     '''
                     err_counter = 0
                     self.dataOut.flagNoData = True
                     if self.__isBufferEmpty():
                         self.__flagDiscontinuousBlock = False
                         while True:
                             if self.__readNextBlock():
                                 break
                             if self.__thisUnixSample > self.__endUTCSecond*self.__sample_rate:
                                 return False
                             if self.__flagDiscontinuousBlock:
                                 print('[Reading] discontinuous block found ... continue with the next block')
                                 continue
                             if not self.__online:
                                 return False
                             err_counter += 1
                             if err_counter > nTries:
                                 return False
                             print('[Reading] waiting %d seconds to read a new block' %seconds)
                             sleep(seconds)
                     self.dataOut.data = self.__data_buffer[:,self.__bufferIndex:self.__bufferIndex+self.__nSamples]
                     self.dataOut.utctime = (self.__thisUnixSample + self.__bufferIndex)/self.__sample_rate
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.__flagDiscontinuousBlock
                     self.dataOut.profileIndex = self.profileIndex
                     self.__bufferIndex += self.__nSamples
                     self.profileIndex += 1
                     if self.profileIndex == self.dataOut.nProfiles:
                         self.profileIndex = 0
                     return True
                 def printInfo(self):
                     '''
                     '''
                     if self.__printInfo == False:
                         return
             #         self.systemHeaderObj.printInfo()
             #         self.radarControllerHeaderObj.printInfo()
                     self.__printInfo = False
                 def printNumberOfBlock(self):
                     '''
                     '''
                     print(self.profileIndex)
                 def run(self, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     '''
                     if not self.isConfig:
                         self.setup(**kwargs)
                     self.getData(seconds=self.__delay)
                     return
             class USRPWriter(Operation):
                 '''
                 classdocs
                 '''
                 def __init__(self, **kwargs):
                     '''
                     Constructor
                     '''
                     Operation.__init__(self, **kwargs)
                     self.dataOut = None
                 def setup(self, dataIn, path, blocksPerFile, set=0, ext=None):
                     '''
                     In this method we should set all initial parameters.
                     Input:
                         dataIn        :        Input data will also be outputa data
                     '''
                     self.dataOut = dataIn
                     self.isConfig = True
                     return
                 def run(self, dataIn, **kwargs):
                     '''
                     This method will be called many times so here you should put all your code
                     Inputs:
                         dataIn        :        object with the data
                     '''
                     if not self.isConfig:
                         self.setup(dataIn, **kwargs)
             if __name__ == '__main__':
                 readObj = USRPReader()
                 while True:
                     readObj.run(path='/Volumes/DATA/haystack/passive_radar/')
             #         readObj.printInfo()
                     readObj.printNumberOfBlock()

schainpy/model/io/pxIO_param.py +1 -3

             '''
             Created on Jan 15, 2018
             @author: Juan C. Espinoza
             '''
             import os
             import sys
             import time
             import glob
             import datetime
             import tarfile
             import numpy
             from .utils import folder_in_range
             from schainpy.model.io.jroIO_base import JRODataReader
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             from schainpy.model.data.jrodata import Parameters
             from schainpy.utils import log
             try:
                 from netCDF4 import Dataset
             except:
-                log.warning(
+                pass
-                    'You should install "netCDF4" module if you want to read/write NCDF files'
             UT1970 = datetime.datetime(1970, 1, 1) - datetime.timedelta(seconds=time.timezone)
             class PXReader(JRODataReader, ProcessingUnit):
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     self.dataOut = Parameters()
                     self.counter_records = 0
                     self.nrecords = None
                     self.flagNoMoreFiles = 0
                     self.isConfig = False
                     self.filename = None
                     self.intervals = set()
                     self.ext = ('.nc', '.tgz')
                     self.online_mode = False
                 def setup(self,
                           path=None,
                           startDate=None,
                           endDate=None,
                           format=None,
                           startTime=datetime.time(0, 0, 0),
                           endTime=datetime.time(23, 59, 59),
                           walk=False,
                           **kwargs):
                     self.path = path
                     self.startDate = startDate
                     self.endDate = endDate
                     self.startTime = startTime
                     self.endTime = endTime
                     self.datatime = datetime.datetime(1900,1,1)
                     self.walk = walk
                     self.nTries = kwargs.get('nTries', 10)
                     self.online = kwargs.get('online', False)
                     self.delay = kwargs.get('delay', 60)
                     self.ele = kwargs.get('ext', '')
                     if self.path is None:
                         raise ValueError('The path is not valid')
                     self.search_files(path, startDate, endDate, startTime, endTime, walk)
                     self.cursor = 0
                     self.counter_records = 0
                     if not self.files:
                         raise  Warning('There is no files matching these date in the folder: {}. \n Check startDate and endDate'.format(path))
                 def search_files(self, path, startDate, endDate, startTime, endTime, walk):
                     '''
                      Searching for NCDF files in path
                      Creating a list of files to procces included in [startDate,endDate]
                      Input:
                          path - Path to find files
                     '''
                     log.log('Searching files {} in {} '.format(self.ext, path), 'PXReader')
                     if walk:
                         paths = [os.path.join(path, p) for p in os.listdir(path) if os.path.isdir(os.path.join(path, p))]
                         paths.sort()
                     else:
                         paths = [path]
                     fileList0 = []
                     for subpath in paths:
                         if not folder_in_range(subpath.split('/')[-1], startDate, endDate, '%Y%m%d'):
                             continue
                         fileList0 += [os.path.join(subpath, s) for s in glob.glob1(subpath, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
                     fileList0.sort()
                     if self.online:
                         fileList0 = fileList0[-1:]
                     self.files = {}
                     startDate = startDate - datetime.timedelta(1)
                     endDate = endDate + datetime.timedelta(1)
                     for fullname in fileList0:
                         thisFile = fullname.split('/')[-1]
                         year = thisFile[3:7]
                         if not year.isdigit():
                             continue
                         month = thisFile[7:9]
                         if not month.isdigit():
                             continue
                         day = thisFile[9:11]
                         if not day.isdigit():
                             continue
                         year, month, day = int(year), int(month), int(day)
                         dateFile = datetime.date(year, month, day)
                         timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
                         if (startDate > dateFile) or (endDate < dateFile):
                             continue
                         dt = datetime.datetime.combine(dateFile, timeFile)
                         if dt not in self.files:
                             self.files[dt] = []
                         self.files[dt].append(fullname)
                     self.dates = list(self.files.keys())
                     self.dates.sort()
                     return
                 def search_files_online(self):
                     '''
                      Searching for NCDF files in online mode path
                      Creating a list of files to procces included in [startDate,endDate]
                      Input:
                          path - Path to find files
                     '''
                     self.files = {}
                     for n in range(self.nTries):
                         if self.walk:
                             paths = [os.path.join(self.path, p) for p in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, p))]
                             paths.sort()
                             path = paths[-1]
                         else:
                             path = self.path
                         new_files = [os.path.join(path, s) for s in glob.glob1(path, '*') if os.path.splitext(s)[-1] in self.ext and '{}'.format(self.ele) in s]
                         new_files.sort()
                         for fullname in new_files:
                             thisFile = fullname.split('/')[-1]
                             year = thisFile[3:7]
                             if not year.isdigit():
                                 continue
                             month = thisFile[7:9]
                             if not month.isdigit():
                                 continue
                             day = thisFile[9:11]
                             if not day.isdigit():
                                 continue
                             year, month, day = int(year), int(month), int(day)
                             dateFile = datetime.date(year, month, day)
                             timeFile = datetime.time(int(thisFile[12:14]), int(thisFile[14:16]), int(thisFile[16:18]))
                             dt = datetime.datetime.combine(dateFile, timeFile)
                             if self.dt >= dt:
                                 continue
                             if dt not in self.files:
                                 self.dt = dt
                                 self.files[dt] = []
                             self.files[dt].append(fullname)
                             break
                         if self.files:
                             break
                         else:
                             log.warning('Waiting {} seconds for the next file, try {} ...'.format(self.delay, n + 1), 'PXReader')
                             time.sleep(self.delay)
                     if not self.files:
                         return 0
                     self.dates = list(self.files.keys())
                     self.dates.sort()
                     self.cursor = 0
                     return 1
                 def parseFile(self):
                     '''
                     '''
                     header = {}
                     for attr in self.fp.ncattrs():
                         header[str(attr)] = getattr(self.fp, attr)
                     self.header.append(header)
                     self.data[header['TypeName']] = numpy.array(self.fp.variables[header['TypeName']])
                 def setNextFile(self):
                     '''
                     Open next files for the current datetime
                     '''
                     cursor = self.cursor
                     if not self.online_mode:
                         if cursor == len(self.dates):
                             if self.online:
                                 cursor = 0
                                 self.dt = self.dates[cursor]
                                 self.online_mode = True
                                 if not self.search_files_online():
                                     log.success('No more files', 'PXReader')
                                     return 0
                             else:
                                 log.success('No more files', 'PXReader')
                                 self.flagNoMoreFiles = 1
                                 return 0
                     else:
                         if not self.search_files_online():
                             return 0
                         cursor = self.cursor
                     self.data = {}
                     self.header = []
                     for fullname in self.files[self.dates[cursor]]:
                         log.log('Opening: {}'.format(fullname), 'PXReader')
                         if os.path.splitext(fullname)[-1] == '.tgz':
                             tar = tarfile.open(fullname, 'r:gz')
                             tar.extractall('/tmp')
                             files = [os.path.join('/tmp', member.name) for member in tar.getmembers()]
                         else:
                             files = [fullname]
                         for filename in files:
                             if self.filename is not None:
                                 self.fp.close()
                             self.filename = filename
                             self.filedate = self.dates[cursor]
                             self.fp = Dataset(self.filename, 'r')
                             self.parseFile()
                     self.counter_records += 1
                     self.cursor += 1
                     return 1
                 def readNextFile(self):
                     while True:
                         self.flagDiscontinuousBlock = 0
                         if not self.setNextFile():
                             return 0
                         self.datatime = datetime.datetime.utcfromtimestamp(self.header[0]['Time'])
                         if self.online:
                             break
                         if (self.datatime < datetime.datetime.combine(self.startDate, self.startTime)) or \
                            (self.datatime > datetime.datetime.combine(self.endDate, self.endTime)):
                             log.warning(
                                 'Reading Record No. {}/{} -> {} [Skipping]'.format(
                                     self.counter_records,
                                     self.nrecords,
                                     self.datatime.ctime()),
                                 'PXReader')
                             continue
                         break
                     log.log(
                         'Reading Record No. {}/{} -> {}'.format(
                             self.counter_records,
                             self.nrecords,
                             self.datatime.ctime()),
                         'PXReader')
                     return 1
                 def set_output(self):
                     '''
                     Storing data from buffer to dataOut object
                     '''
                     self.data['Elevation'] = numpy.array(self.fp.variables['Elevation'])
                     self.data['Azimuth'] = numpy.array(self.fp.variables['Azimuth'])
                     self.dataOut.range = numpy.array(self.fp.variables['GateWidth'])
                     self.dataOut.data = self.data
                     self.dataOut.units = [h['Unit-value'] for h in self.header]
                     self.dataOut.parameters = [h['TypeName'] for h in self.header]
                     self.dataOut.missing = self.header[0]['MissingData']
                     self.dataOut.max_range = self.header[0]['MaximumRange-value']
                     self.dataOut.elevation = self.header[0]['Elevation']
                     self.dataOut.azimuth = self.header[0]['Azimuth']
                     self.dataOut.latitude = self.header[0]['Latitude']
                     self.dataOut.longitude = self.header[0]['Longitude']
                     self.dataOut.utctime = self.header[0]['Time']
                     self.dataOut.utctimeInit = self.dataOut.utctime
                     self.dataOut.useLocalTime = True
                     self.dataOut.flagNoData = False
                     self.dataOut.flagDiscontinuousBlock = self.flagDiscontinuousBlock
                     log.log('Parameters found: {}'.format(','.join(self.dataOut.parameters)),
                                 'PXReader')
                 def getData(self):
                     '''
                     Storing data from databuffer to dataOut object
                     '''
                     if self.flagNoMoreFiles:
                         self.dataOut.flagNoData = True
                         log.error('No file left to process', 'PXReader')
                         return 0
                     if not self.readNextFile():
                         self.dataOut.flagNoData = True
                         return 0
                     self.set_output()
                     return 1

schainpy/model/utils/jroutils_ftp.py +1 -1

             '''
             @author: Daniel Suarez
             '''
             import os
             import glob
             import ftplib
             try:
                 import paramiko
                 import scp
             except:
-                print("You should install paramiko and scp libraries \nif you want to use SSH protocol to upload files to the server")
+                pass
             import time
             import threading
             Thread = threading.Thread
             # try:
             #     from gevent import sleep
             # except:
             from time import sleep
             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
             class Remote(Thread):
                 """
                 Remote is a parent class used to define the behaviour of FTP and SSH class. These clases are
                 used to upload or download files remotely.
                 Non-standard Python modules used:
                     None
                 Written by:
                     "Miguel Urco":mailto:miguel.urco@jro.igp.gob.pe Jun. 03, 2015
                 """
                 server = None
                 username = None
                 password = None
                 remotefolder = None
                 period = 60
                 fileList = []
                 bussy = False
                 def __init__(self, server, username, password, remotefolder, period=60):
                     Thread.__init__(self)
                     self.setDaemon(True)
                     self.status = 0
                     self.__server = server
                     self.__username = username
                     self.__password = password
                     self.__remotefolder = remotefolder
                     self.period = period
                     self.fileList = []
                     self.bussy = False
                     self.stopFlag = False
                     print("[Remote Server] Opening server: %s" %self.__server)
                     if self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                         print("[Remote Server] %s server was opened successfully" %self.__server)
                     self.close()
                     self.mutex = threading.Lock()
                 def stop(self):
                     self.stopFlag = True
                     self.join(10)
                 def open(self):
                     """
                     Connect to server and create a connection class (FTP or SSH) to remote server.
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def close(self):
                     """
                     Close connection to server
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def mkdir(self, remotefolder):
                     """
                     Create a folder remotely
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def cd(self, remotefolder):
                     """
                     Change working directory in remote server
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def download(self, filename, localfolder=None):
                     """
                     Download a file from server to local host
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def sendFile(self, fullfilename):
                     """
                     sendFile method is used to upload a local file to the current directory in remote server
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                     Returns:
 in error case else 1
                     """
                     raise NotImplementedError("Implement this method in child class")
                 def upload(self, fullfilename, remotefolder=None):
                     """
                     upload method is used to upload a local file to remote directory. This method changes
                     working directory before sending a file.
                     Inputs:
                         fullfilename        - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
 in error case else 1
                     """
                     print("[Remote Server] Uploading %s to %s:%s" %(fullfilename, self.server, self.remotefolder))
                     if not self.status:
                         return 0
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     if not self.cd(remotefolder):
                         return 0
                     if not self.sendFile(fullfilename):
                         print("[Remote Server] Error uploading file %s" %fullfilename)
                         return 0
                     print("[Remote Server] upload finished successfully")
                     return 1
                 def delete(self, filename):
                     """
                     Remove a file from remote server
                     """
                     pass
                 def updateFileList(self, fileList):
                     """
                     Remove a file from remote server
                     """
                     if fileList == self.fileList:
                         return 0
                     self.mutex.acquire()
                     #         init = time.time()
                     #
                     #         while(self.bussy):
                     #             sleep(0.1)
                     #             if time.time() - init > 2*self.period:
                     #                 return 0
                     self.fileList = fileList
                     self.mutex.release()
                     return 1
                 def run(self):
                     if not self.status:
                         print("Finishing FTP service")
                         return
                     if not self.cd(self.remotefolder):
                         raise ValueError("Could not access to the new remote directory: %s" %self.remotefolder)
                     while True:
                         for i in range(self.period):
                             if self.stopFlag:
                                 break
                             sleep(1)
                         if self.stopFlag:
                             break
                         #   self.bussy = True
                         self.mutex.acquire()
                         print("[Remote Server] Opening %s" %self.__server)
                         if not self.open(self.__server, self.__username, self.__password, self.__remotefolder):
                             self.mutex.release()
                             continue
                         for thisFile in self.fileList:
                             self.upload(thisFile, self.remotefolder)
                         print("[Remote Server] Closing %s" %self.__server)
                         self.close()
                         self.mutex.release()
                         # self.bussy = False
                     print("[Remote Server] Thread stopped successfully")
             class FTPClient(Remote):
                 __ftpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder):
                     """
                     This method is used to set FTP parameters and establish a connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return:
                         Boolean     -     Returns 1 if a connection has been established, 0 otherwise
                     Affects:
                         self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     if server == None:
                         raise ValueError("FTP server should be defined")
                     if username == None:
                         raise ValueError("FTP username should be defined")
                     if password == None:
                         raise ValueError("FTP password should be defined")
                     if remotefolder == None:
                         raise ValueError("FTP remote folder should be defined")
                     try:
                         ftpClientObj = ftplib.FTP(server)
                     except ftplib.all_errors as e:
                         print("[FTP Server]: FTP server connection fail: %s" %server)
                         print("[FTP Server]:", e)
                         self.status = 0
                         return 0
                     try:
                         ftpClientObj.login(username, password)
                     except ftplib.all_errors:
                         print("[FTP Server]: FTP username or password are incorrect")
                         self.status = 0
                         return 0
                     if remotefolder == None:
                         remotefolder = ftpClientObj.pwd()
                     else:
                         try:
                             ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             print("[FTP Server]: FTP remote folder is invalid: %s" %remotefolder)
                             remotefolder = ftpClientObj.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.__ftpClientObj = ftpClientObj
                     self.status = 1
                     return 1
                 def close(self):
                     """
                     Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__ftpClientObj.close()
                 def mkdir(self, remotefolder):
                     """
                     mkdir is used to make a new directory in remote server
                     Input:
                         remotefolder    - directory name
                     Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     try:
                         self.__ftpClientObj.mkd(dirname)
                     except ftplib.all_errors:
                         print("[FTP Server]: Error creating remote folder: %s" %remotefolder)
                         return 0
                     return 1
                 def cd(self, remotefolder):
                     """
                     cd is used to change remote working directory on server
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.remotefolder
                     Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     try:
                         self.__ftpClientObj.cwd(remotefolder)
                     except ftplib.all_errors:
                         print('[FTP Server]: Error changing to %s' %remotefolder)
                         print('[FTP Server]: Trying to create remote folder')
                         if not self.mkdir(remotefolder):
                             print('[FTP Server]: Remote folder could not be created')
                             return 0
                         try:
                             self.__ftpClientObj.cwd(remotefolder)
                         except ftplib.all_errors:
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     fp = open(fullfilename, 'rb')
                     filename = os.path.basename(fullfilename)
                     command = "STOR %s" %filename
                     try:
                         self.__ftpClientObj.storbinary(command, fp)
                     except ftplib.all_errors as e:
                         print("[FTP Server]:", e)
                         return 0
                     try:
                         self.__ftpClientObj.sendcmd('SITE CHMOD 755 ' + filename)
                     except ftplib.all_errors as e:
                         print("[FTP Server]:", e)
                     fp.close()
                     return 1
             class SSHClient(Remote):
                 __sshClientObj = None
                 __scpClientObj = None
                 def __init__(self, server, username, password, remotefolder, period=60):
                     """
                     """
                     Remote.__init__(self, server, username, password, remotefolder, period)
                 def open(self, server, username, password, remotefolder, port=22):
                     """
                         This method is used to set SSH parameters and establish a connection to a remote server
                         Inputs:
                             server    - remote server IP Address
                             username    - remote server Username
                             password    - remote server password
                             remotefolder    - remote server current working directory
                         Return: void
                         Affects:
                             self.status        - in case of error or fail connection this parameter is set to 0 else 1
                     """
                     import socket
                     if server == None:
                         raise ValueError("SSH server should be defined")
                     if username == None:
                         raise ValueError("SSH username should be defined")
                     if password == None:
                         raise ValueError("SSH password should be defined")
                     if remotefolder == None:
                         raise ValueError("SSH remote folder should be defined")
                     sshClientObj = paramiko.SSHClient()
                     sshClientObj.load_system_host_keys()
                     sshClientObj.set_missing_host_key_policy(paramiko.WarningPolicy())
                     self.status = 0
                     try:
                         sshClientObj.connect(server, username=username, password=password, port=port)
                     except paramiko.AuthenticationException as e:
                 #             print "SSH username or password are incorrect: %s"
                         print("[SSH Server]:", e)
                         return 0
                     except SSHException as e:
                         print("[SSH Server]:", e)
                         return 0
                     except socket.error:
                         self.status = 0
                         print("[SSH Server]:", e)
                         return 0
                     self.status = 1
                     scpClientObj = scp.SCPClient(sshClientObj.get_transport(), socket_timeout=30)
                     if remotefolder == None:
                         remotefolder = self.pwd()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.__sshClientObj = sshClientObj
                     self.__scpClientObj = scpClientObj
                     self.status = 1
                     if not self.cd(remotefolder):
                         raise ValueError("[SSH Server]: Could not access to remote folder: %s" %remotefolder)
                         return 0
                     self.remotefolder = remotefolder
                     return 1
                 def close(self):
                     """
                         Close connection to remote server
                     """
                     if not self.status:
                         return 0
                     self.__scpClientObj.close()
                     self.__sshClientObj.close()
                 def __execute(self, command):
                     """
                         __execute a command on remote server
                         Input:
                             command    - Exmaple 'ls -l'
                         Return:
 in error case else 1
                     """
                     if not self.status:
                         return 0
                     stdin, stdout, stderr = self.__sshClientObj.exec_command(command)
                     result = stderr.readlines()
                     if len(result) > 1:
                         return 0
                     result = stdout.readlines()
                     if len(result) > 1:
                         return result[0][:-1]
                     return 1
                 def mkdir(self, remotefolder):
                     """
                         mkdir is used to make a new directory in remote server
                         Input:
                             remotefolder    - directory name
                         Return:
 in error case else 1
                     """
                     command = 'mkdir %s' %remotefolder
                     return self.__execute(command)
                 def pwd(self):
                     command = 'pwd'
                     return self.__execute(command)
                 def cd(self, remotefolder):
                     """
                         cd is used to change remote working directory on server
                         Input:
                             remotefolder    - current working directory
                         Affects:
                             self.remotefolder
                         Return:
 in case of error else 1
                     """
                     if not self.status:
                         return 0
                     if remotefolder == self.remotefolder:
                         return 1
                     chk_command = "cd %s; pwd" %remotefolder
                     mkdir_command = "mkdir %s" %remotefolder
                     if not self.__execute(chk_command):
                         if not self.__execute(mkdir_command):
                             self.remotefolder = None
                             return 0
                     self.remotefolder = remotefolder
                     return 1
                 def sendFile(self, fullfilename):
                     if not self.status:
                         return 0
                     try:
                         self.__scpClientObj.put(fullfilename, remote_path=self.remotefolder)
                     except scp.ScpError as e:
                         print("[SSH Server]", str(e))
                         return 0
                     remotefile = os.path.join(self.remotefolder, os.path.split(fullfilename)[-1])
                     command = 'chmod 775 %s' %remotefile
                     return self.__execute(command)
             class SendToServer(ProcessingUnit):
                 def __init__(self, **kwargs):
                     ProcessingUnit.__init__(self, **kwargs)
                     self.isConfig = False
                     self.clientObj = None
                 def setup(self, server, username, password, remotefolder, localfolder, ext='.png', period=60, protocol='ftp', **kwargs):
                     self.clientObj = None
                     self.localfolder = localfolder
                     self.ext = ext
                     self.period = period
                     if str.lower(protocol) == 'ftp':
                         self.clientObj = FTPClient(server, username, password, remotefolder, period)
                     if str.lower(protocol) == 'ssh':
                         self.clientObj = SSHClient(server, username, password, remotefolder, period)
                     if not self.clientObj:
                         raise ValueError("%s has been chosen as remote access protocol but it is not valid" %protocol)
                     self.clientObj.start()
                 def findFiles(self):
                     if not type(self.localfolder) == list:
                         folderList = [self.localfolder]
                     else:
                         folderList = self.localfolder
                     #Remove duplicate items
                     folderList = list(set(folderList))
                     fullfilenameList = []
                     for thisFolder in folderList:
                         print("[Remote Server]: Searching files on %s" %thisFolder)
                         filenameList = glob.glob1(thisFolder, '*%s' %self.ext)
                         if len(filenameList) < 1:
                             continue
                         for thisFile in filenameList:
                             fullfilename = os.path.join(thisFolder, thisFile)
                             if fullfilename in fullfilenameList:
                                 continue
                             #Only files modified in the last 30 minutes are considered
                             if os.path.getmtime(fullfilename) < time.time() - 30*60:
                                 continue
                             fullfilenameList.append(fullfilename)
                     return fullfilenameList
                 def run(self, **kwargs):
                     if not self.isConfig:
                         self.init = time.time()
                         self.setup(**kwargs)
                         self.isConfig = True
                     if not self.clientObj.is_alive():
                         print("[Remote Server]: Restarting connection ")
                         self.setup(**kwargs)
                     if time.time() - self.init >= self.period:
                         fullfilenameList = self.findFiles()
                         if self.clientObj.updateFileList(fullfilenameList):
                             print("[Remote Server]: Sending the next files ", str(fullfilenameList))
                         self.init = time.time()
                 def close(self):
                     print("[Remote Server] Stopping thread")
                     self.clientObj.stop()
             class FTP(object):
                 """
                 Ftp is a public class used to define custom File Transfer Protocol from "ftplib" python module
                 Non-standard Python modules used: None
                 Written by "Daniel Suarez":mailto:daniel.suarez@jro.igp.gob.pe  Oct. 26, 2010
                 """
                 def __init__(self,server = None, username=None, password=None, remotefolder=None):
                     """
                     This method is used to setting parameters for FTP and establishing connection to remote server
                     Inputs:
                         server    - remote server IP Address
                         username    - remote server Username
                         password    - remote server password
                         remotefolder    - remote server current working directory
                     Return: void
                     Affects:
                         self.status    - in Error Case or Connection Failed this parameter is set to 1 else 0
                         self.folderList    - sub-folder list of remote folder
                         self.fileList    - file list of remote folder
                     """
                     if ((server == None) and (username==None) and (password==None) and (remotefolder==None)):
                         server, username, password, remotefolder = self.parmsByDefault()
                     self.server = server
                     self.username = username
                     self.password = password
                     self.remotefolder = remotefolder
                     self.file = None
                     self.ftp = None
                     self.status = 0
                     try:
                         self.ftp = ftplib.FTP(self.server)
                         self.ftp.login(self.username,self.password)
                         self.ftp.cwd(self.remotefolder)
                         #   print 'Connect to FTP Server: Successfully'
                     except ftplib.all_errors:
                         print('Error FTP Service')
                         self.status = 1
                         return
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm as resp:
                         if str(resp) == "550 No files found":
                                 print("no files in this directory")
                                 self.status = 1
                                 return
                     except ftplib.all_errors:
                         print('Error Displaying Dir-Files')
                         self.status = 1
                         return
                     self.fileList = []
                     self.folderList = []
                     #only for test
                     for f in self.dirList:
                         name, ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
                 #                print 'filename: %s - size: %d'%(f,self.ftp.size(f))
                 def parmsByDefault(self):
                     server = 'jro-app.igp.gob.pe'
                     username = 'wmaster'
                     password = 'mst2010vhf'
                     remotefolder = '/home/wmaster/graficos'
                     return server, username, password, remotefolder
                 def mkd(self,dirname):
                     """
                     mkd is used to make directory in remote server
                     Input:
                         dirname    - directory name
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.mkd(dirname)
                     except:
                         print('Error creating remote folder:%s'%dirname)
                         return 1
                     return 0
                 def delete(self,filename):
                     """
                     delete is used to delete file in current working directory of remote server
                     Input:
                         filename    - filename to delete in remote folder
                     Return:
 in error case else 0
                     """
                     try:
                         self.ftp.delete(filename)
                     except:
                         print('Error deleting remote file:%s'%filename)
                         return 1
                     return 0
                 def download(self,filename,localfolder):
                     """
                     download is used to downloading file from remote folder into local folder
                     Inputs:
                         filename    - filename to donwload
                         localfolder    - directory local to store filename
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     self.status = 0
                     if not(filename in self.fileList):
                         print('filename:%s not exists'%filename)
                         self.status = 1
                         return self.status
                     newfilename = os.path.join(localfolder,filename)
                     self.file = open(newfilename, 'wb')
                     try:
                         print('Download: ' + filename)
                         self.ftp.retrbinary('RETR ' + filename, self.__handleDownload)
                         print('Download Complete')
                     except ftplib.all_errors:
                         print('Error Downloading ' + filename)
                         self.status = 1
                         return self.status
                     self.file.close()
                     return self.status
                 def __handleDownload(self,block):
                     """
                     __handleDownload is used to handle writing file
                     """
                     self.file.write(block)
                 def upload(self,filename,remotefolder=None):
                     """
                     upload is used to uploading local file to remote directory
                     Inputs:
                         filename    - full path name of local file to store in remote directory
                         remotefolder    - remote directory
                     Returns:
                         self.status    - 1 in error case else 0
                     """
                     if remotefolder == None:
                         remotefolder = self.remotefolder
                     self.status = 0
                     try:
                         self.ftp.cwd(remotefolder)
                         self.file = open(filename, 'rb')
                         (head, tail) = os.path.split(filename)
                         command = "STOR " + tail
                         print('Uploading: ' + tail)
                         self.ftp.storbinary(command, self.file)
                         print('Upload Completed')
                     except ftplib.all_errors:
                         print('Error Uploading ' + tail)
                         self.status = 1
                         return self.status
                     self.file.close()
                     #back to initial directory in __init__()
                     self.ftp.cwd(self.remotefolder)
                     return self.status
                 def dir(self,remotefolder):
                     """
                     dir is used to change working directory of remote server and get folder and file list
                     Input:
                         remotefolder    - current working directory
                     Affects:
                         self.fileList    - file list of working directory
                     Return:
                         infoList    - list with filenames and size of file in bytes
                         self.folderList    -    folder list
                     """
                     self.remotefolder = remotefolder
                     print('Change to ' + self.remotefolder)
                     try:
                         self.ftp.cwd(remotefolder)
                     except ftplib.all_errors:
                         print('Error Change to ' + self.remotefolder)
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     self.dirList = []
                     try:
                         self.dirList = self.ftp.nlst()
                     except ftplib.error_perm as resp:
                         if str(resp) == "550 No files found":
                                 print("no files in this directory")
                                 infoList = None
                                 self.folderList = None
                                 return infoList,self.folderList
                     except ftplib.all_errors:
                         print('Error Displaying Dir-Files')
                         infoList = None
                         self.folderList = None
                         return infoList,self.folderList
                     infoList = []
                     self.fileList = []
                     self.folderList = []
                     for f in self.dirList:
                         name,ext = os.path.splitext(f)
                         if ext != '':
                             self.fileList.append(f)
                             value = (f,self.ftp.size(f))
                             infoList.append(value)
                         if ext == '':
                             self.folderList.append(f)
                     return infoList,self.folderList
                 def close(self):
                     """
                     close is used to close and end FTP connection
                     Inputs: None
                     Return: void
                     """
                     self.ftp.close()
             class SendByFTP(Operation):
                 def __init__(self, **kwargs):
                     Operation.__init__(self, **kwargs)
                     self.status = 1
                     self.counter = 0
                 def error_print(self, ValueError):
                     print(ValueError, 'Error FTP')
                     print("don't worry the program is running...")
                 def worker_ftp(self, server, username, password, remotefolder, filenameList):
                     self.ftpClientObj = FTP(server, username, password, remotefolder)
                     for filename in filenameList:
                         self.ftpClientObj.upload(filename)
                     self.ftpClientObj.close()
                 def ftp_thread(self, server, username, password, remotefolder):
                     if not(self.status):
                         return
                     import multiprocessing
                     p = multiprocessing.Process(target=self.worker_ftp, args=(server, username, password, remotefolder, self.filenameList,))
                     p.start()
                     p.join(3)
                     if p.is_alive():
                         p.terminate()
                         p.join()
                         print('killing ftp process...')
                         self.status = 0
                         return
                     self.status = 1
                     return
                 def filterByExt(self, ext, localfolder):
                     fnameList = glob.glob1(localfolder,ext)
                     self.filenameList = [os.path.join(localfolder,x) for x in fnameList]
                     if len(self.filenameList) == 0:
                         self.status = 0
                 def run(self, dataOut, ext, localfolder, remotefolder, server, username, password, period=1):
                     self.counter += 1
                     if self.counter >= period:
                         self.filterByExt(ext, localfolder)
                         self.ftp_thread(server, username, password, remotefolder)
                         self.counter = 0
                     self.status = 1

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages