schain-jc Commit - r1179:6414333e2ace

Multiprocessing for writing Units(Spectral, Voltage and Parameters)

George Yong -

r1179:6414333e2ace

parent child

Context file:

r1179:6414333e2ace

Collapse all files

schainpy/model/data/jrodata.py +31 -14

@@ -65,21 +65,38 def hildebrand_sekhon(data, navg):
65	mean : noise's level	65	mean : noise's level
66	"""	66	"""
67		67
68	sort~~ed_spectrum~~ = numpy.sort(data, axis=None)	68	sortdata = numpy.sort(data, axis=None)
69	nnoise = len(sorted_spectrum) # default to all points in the spectrum as noise	69	lenOfData = len(sortdata)
70	for npts in range(1, len(sorted_spectrum)+1):	70	nums_min = lenOfData*0.2
71	partial = sorted_spectrum[:npts]	71
72	mean = partial.mean()	72	if nums_min <= 5:
73	var = partial.var()	73
74	if var * navg < mean**2.:	74	nums_min = 5
75	nnoise = npts	75
76	else:	76	sump = 0.
77	# partial spectrum no longer has characteristics of white noise	77	sumq = 0.
78	break	78
		79	j = 0
		80	cont = 1
		81
		82	while((cont==1)and(j<lenOfData)):
		83
		84	sump += sortdata[j]
		85	sumq += sortdata[j]**2
		86
		87	if j > nums_min:
		88	rtest = float(j)/(j-1) + 1.0/navg
		89	if ((sumqj) > (rtestsump**2)):
		90	j = j - 1
		91	sump = sump - sortdata[j]
		92	sumq = sumq - sortdata[j]**2
		93	cont = 0
		94
		95	j += 1
		96
		97	lnoise = sump /j
79		98
80	noise_spectrum = sorted_spectrum[:nnoise]	99	return lnoise
81	mean = noise_spectrum.mean()
82	return mean
83		100
84		101
85	class Beam:	102	class Beam:

schainpy/model/graphics/jroplot_parameters.py +18 -11

             import inspect
             from .figure import Figure, isRealtime, isTimeInHourRange
             from .plotting_codes import *
+            from schainpy.model.proc.jroproc_base import MPDecorator
+            from schainpy.utils import log
             class FitGauPlot(Figure):
                 WIDTHPROF = None
                 HEIGHTPROF = None
                 PREFIX = 'prm'
-                def __init__(self, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self, **kwargs)
+                    Figure.__init__(self)
                     self.isConfig = False
                     self.__nsubplots = 1
                               thisDatetime=thisDatetime)
             class SkyMapPlot(Figure):
                 __isConfig = None
                         self.isConfig = False
                         update_figfile = True
+            @MPDecorator
             class ParametersPlot(Figure):
                 __isConfig = None
                 nplots = None
                 nchan = None
-                def __init__(self, **kwargs):
+                def __init__(self):#, **kwargs):
-                    Figure.__init__(self, **kwargs)
+                    Figure.__init__(self)#, **kwargs)
                     self.timerange = None
                     self.isConfig = False
                     self.__nsubplots = 1
                         zmin            :    None,
                         zmax            :    None
                     """
+                    if dataOut.flagNoData:
+                        return dataOut
                     if HEIGHT is not None:
                         self.HEIGHT = HEIGHT
                               thisDatetime=thisDatetime,
                               update_figfile=update_figfile)
+                    return dataOut
+            @MPDecorator
             class Parameters1Plot(Figure):
                 __isConfig = None
                 HEIGHTPROF = None
                 PREFIX = 'prm'
-                def __init__(self, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self, **kwargs)
+                    Figure.__init__(self)
                     self.timerange = 2*60*60
                     self.isConfig = False
                     self.__nsubplots = 1
                         zmin            :    None,
                         zmax            :    None
                     """
+                    if dataOut.flagNoData:
+                        return dataOut
                     data_param = getattr(dataOut, parameterObject)
                               wr_period=wr_period,
                               thisDatetime=thisDatetime,
                               update_figfile=False)
+                    return dataOut
             class SpectralFittingPlot(Figure):
                               ftp=ftp,
                               wr_period=wr_period,
                               thisDatetime=thisDatetime)
+  No newline at end of file

schainpy/model/graphics/jroplot_spectra.py +17 -17

                 HEIGHTPROF = None
                 PREFIX = 'spc'
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self)#, **kwargs)
+                    Figure.__init__(self)
                     self.isConfig = False
                     self.__nsubplots = 1
                     self.WIDTH = 250
                 HEIGHTPROF = None
                 PREFIX = 'cspc'
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self)#, **kwargs)
+                    Figure.__init__(self)
                     self.isConfig = False
                     self.__nsubplots = 4
                     self.counter_imagwr = 0
                 HEIGHTPROF = None
                 PREFIX = 'rti'
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self)#, **kwargs)
+                    Figure.__init__(self)
                     self.timerange = None
                     self.isConfig = False
                     self.__nsubplots = 1
                 HEIGHTPROF = None
                 PREFIX = 'cmap'
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self)#, **kwargs)
+                    Figure.__init__(self)
                     self.timerange = 2*60*60
                     self.isConfig = False
                     self.__nsubplots = 1
                 HEIGHTPROF = None
                 PREFIX = 'spcprofile'
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self)#, **kwargs)
+                    Figure.__init__(self)
                     self.isConfig = False
                     self.__nsubplots = 1
                 HEIGHTPROF = None
                 PREFIX = 'spc_cut'
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self)#, **kwargs)
+                    Figure.__init__(self)
                     self.isConfig = False
                     self.__nsubplots = 1
                 PREFIX = 'noise'
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self)#, **kwargs)
+                    Figure.__init__(self)
                     self.timerange = 24*60*60
                     self.isConfig = False
                     self.__nsubplots = 1
                 PREFIX = 'beacon_phase'
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Figure.__init__(self)#, **kwargs)
+                    Figure.__init__(self)
                     self.timerange = 24*60*60
                     self.isConfig = False
                     self.__nsubplots = 1
                               thisDatetime=thisDatetime,
                               update_figfile=update_figfile)
-                    return dataOut  #Yong
  No newline at end of file
+                    return dataOut
  No newline at end of file

schainpy/model/graphics/jroplot_voltage.py 0 0

NO CONTENT: modified file

schainpy/model/io/jroIO_base.py +2 0

                                    set=set, ext=ext, datatype=datatype, **kwargs)
                         self.isConfig = True
+                    self.dataOut = dataOut
                     self.putData()
+                    return self.dataOut
  No newline at end of file

schainpy/model/io/jroIO_param.py +24 -16

             import datetime
             from schainpy.model.data.jrodata import *
-            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation
+            from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             # from .jroIO_base import *
             from schainpy.model.io.jroIO_base import *
             import schainpy
+            from schainpy.utils import log
+            @MPDecorator
-            class ParamReader(ProcessingUnit):
+            class ParamReader(JRODataReader,ProcessingUnit):
                 '''
                 Reads HDF5 format files
                 dataOut = None
-                def __init__(self, **kwargs):
+                def __init__(self):#, **kwargs):
-                    ProcessingUnit.__init__(self, **kwargs)
+                    ProcessingUnit.__init__(self) #, **kwargs)
                     self.dataOut = Parameters()
                     return
                     self.getData()
                     return
+            @MPDecorator
             class ParamWriter(Operation):
                 '''
                 HDF5 Writer, stores parameters data in HDF5 format files
                 lastTime = None
-                def __init__(self, **kwargs):
+                def __init__(self):#, **kwargs):
-                    Operation.__init__(self, **kwargs)
+                    Operation.__init__(self)#, **kwargs)
-                    self.isConfig = False
+                    #self.isConfig = False
                     return
                 def setup(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
                     self.dataList = dataList
                     self.dataOut = dataOut
                     self.mode = mode
                     if self.mode is not None:
                         self.mode = numpy.zeros(len(self.dataList)) + mode
                     else:
+                        #self.mode = numpy.ones(len(self.dataList),int)
                         self.mode = numpy.ones(len(self.dataList))
+                    log.error(self.mode)#yong
                     arrayDim = numpy.zeros((len(self.dataList),5))
                         #Not array, just a number
                         #Mode 0
+                        #log.error(mode)#yong
+                        #log.error(len(mode))#yong
+                        #log.error(type(mode))#yong
                         if type(dataAux)==float or type(dataAux)==int:
                             dsDict['mode'] = 0
                             dsDict['nDim'] = 0
                             dsList.append(dsDict)
                         #Mode 2: meteors
-                        elif mode[i] == 2:
+                        elif self.mode[i] == 2:
             #                 dsDict['nDim'] = 0
                             dsDict['dsName'] = 'table0'
                             dsDict['mode'] = 2      # Mode meteors
                             dsDict['dsNumber'] = 1
                             arrayDim[i,3] = dataAux.shape[-1]
-                            arrayDim[i,4] = mode[i]         #Mode the data was stored
+                            arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             dsList.append(dsDict)
                         else:
                             arrayDim0 = dataAux.shape       #Data dimensions
                             arrayDim[i,0] = len(arrayDim0)  #Number of array dimensions
-                            arrayDim[i,4] = mode[i]         #Mode the data was stored
+                            arrayDim[i,4] = self.mode[i]         #Mode the data was stored
                             strtable = 'table'
                             dsDict['mode'] = 1      # Mode parameters
                         os.makedirs(fullpath)
                         setFile = -1 #inicializo mi contador de seteo
-                    if self.setType is None:
+                    if None is None:
                         setFile += 1
                         file = '%s%4.4d%3.3d%03d%s' % (self.metaoptchar,
                                                        timeTuple.tm_year,
                             for j in range(dsInfo['dsNumber']):
                                 dsInfo = dsList[i]
                                 tableName = dsInfo['dsName']
-                                shape = int(dsInfo['shape'])
                                 if dsInfo['nDim'] == 3:
+                                    shape = dsInfo['shape'].astype(int)
                                     ds0 = grp0.create_dataset(tableName, (shape[0],shape[1],1) , data = numpy.zeros((shape[0],shape[1],1)), maxshape = (None,shape[1],None), chunks=True)
                                 else:
+                                    shape = int(dsInfo['shape'])
                                     ds0 = grp0.create_dataset(tableName, (1,shape), data = numpy.zeros((1,shape)) , maxshape=(None,shape), chunks=True)
                                 ds.append(ds0)
                     self.fp.close()
                     return
-                def run(self, dataOut, path=None, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
+                def run(self, dataOut, path, blocksPerFile=10, metadataList=None, dataList=None, mode=None, **kwargs):
                     if not(self.isConfig):
                         flagdata = self.setup(dataOut, path=path, blocksPerFile=blocksPerFile,
                     self.putData()
                     return
+  No newline at end of file

schainpy/model/io/jroIO_spectra.py +5 -6

             from schainpy.model.proc.jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jroheaderIO import PROCFLAG, BasicHeader, SystemHeader, RadarControllerHeader, ProcessingHeader
             from schainpy.model.data.jrodata import Spectra
+            from schainpy.utils import log
             @MPDecorator
             class SpectraReader(JRODataReader, ProcessingUnit):
                     self.dataOut.realtime = self.online
                     return self.dataOut.data_spc
+            @MPDecorator
             class SpectraWriter(JRODataWriter, Operation):
                 """
             #    dataOut = None
-                def __init__(self, **kwargs):
+                def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase SpectraWriter para la escritura de datos de espectros.
                     Return: None
                     """
-                    Operation.__init__(self, **kwargs)
+                    Operation.__init__(self)#, **kwargs)
-                    self.isConfig = False
+                    #self.isConfig = False
                     self.nTotalBlocks = 0
             #            self.setFirstHeader()
                         self.writeNextBlock()
-                    return 1
                 def __getBlockSize(self):
                     '''
                     Este metodos determina el cantidad de bytes para un bloque de datos de tipo Spectra

schainpy/model/io/jroIO_voltage.py +3 -3

                     return self.dataOut.data
+            @MPDecorator
             class VoltageWriter(JRODataWriter, Operation):
                 """
                 Esta clase permite escribir datos de voltajes a archivos procesados (.r). La escritura
                 shapeBuffer = None
-                def __init__(self, **kwargs):
+                def __init__(self):#, **kwargs):
                     """
                     Inicializador de la clase VoltageWriter para la escritura de datos de espectros.
                     Return: None
                     """
-                    Operation.__init__(self, **kwargs)
+                    Operation.__init__(self)#, **kwargs)
                     self.nTotalBlocks = 0

schainpy/model/proc/jroproc_base.py 0 0

NO CONTENT: modified file

schainpy/model/proc/jroproc_parameters.py +8 -3

             from scipy.optimize import fmin_l_bfgs_b #optimize with bounds on state papameters
-            from .jroproc_base import ProcessingUnit, Operation
+            from .jroproc_base import ProcessingUnit, Operation, MPDecorator
             from schainpy.model.data.jrodata import Parameters, hildebrand_sekhon
             from scipy import asarray as ar,exp
             from scipy.optimize import curve_fit
+            from schainpy.utils import log
             import warnings
             from numpy import NaN
             from scipy.optimize.optimize import OptimizeWarning
                         break
                 return func.__get__(obj, cls)
+            @MPDecorator
             class ParametersProc(ProcessingUnit):
+                METHODS = {}
                 nSeconds = None
                 def __init__(self):
                     self.firstdatatime = None
                     self.profIndex = 0
                     self.dataOut = Parameters()
+                    self.setupReq = False #Agregar a todas las unidades de proc
                 def __updateObjFromInput(self):
                 def run(self):
                     #----------------------    Voltage Data    ---------------------------
                     if self.dataIn.type == "Voltage":
                     dataOut.data_DOP = data_param[:,1]
                     dataOut.data_MEAN = data_param[:,2]
                     dataOut.data_STD = data_param[:,3]
-                    return
+                    return dataOut
                 def __calculateMoments(self, oldspec, oldfreq, n0,
                                        nicoh = None, graph = None, smooth = None, type1 = None, fwindow = None, snrth = None, dc = None, aliasing = None, oldfd = None, wwauto = None):

schainpy/model/proc/jroproc_spectra.py +5 -18

             @MPDecorator
             class SpectraProc(ProcessingUnit):
-                METHODS = {'selectHeights'        : ['minHei', 'maxHei'],
-                           'selectChannels'       : 'channelList',
-                           'selectChannelsByIndex': 'channelIndexList',
-                           'getBeaconSignal'      : ['tauindex', 'channelindex', 'hei_ref'],
-                           'selectHeightsByIndex' : ['minIndex', 'maxIndex']
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    ProcessingUnit.__init__(self)#, **kwargs)
+                    ProcessingUnit.__init__(self)
                     self.buffer = None
                     self.firstdatatime = None
                     if self.dataIn.type == "Spectra":
                         self.dataOut.copy(self.dataIn)
-                        # if not pairsList:
-                        #     pairsList = itertools.combinations(self.dataOut.channelList, 2)
-                        # if self.dataOut.data_cspc is not None:
-                        #     self.__selectPairs(pairsList)
                         if shift_fft:
                             #desplaza a la derecha en el eje 2 determinadas posiciones
                             shift = int(self.dataOut.nFFTPoints/2)
                 n = None
-                def __init__(self):#, **kwargs):
+                def __init__(self):
-                    Operation.__init__(self)#, **kwargs)
+                    Operation.__init__(self)
-            #         self.isConfig = False
                 def setup(self, n=None, timeInterval=None, overlapping=False):
                     """
                     if n is not None:
                         self.n = int(n)
                     else:
-                        # if (type(timeInterval)!=integer) -> change this line
                         self.__integrationtime = int(timeInterval)
                         self.n = None
                         self.__byTime = True

schainpy/model/proc/jroproc_voltage.py +15 -15

                 __dataToPutStride = False
                 n = None
-                def __init__(self):#, **kwargs):
+                def __init__(self, **kwargs):
-                    Operation.__init__(self)#, **kwargs)
+                    Operation.__init__(self, **kwargs)
                     #   self.isConfig = False
                 nCode = None
                 nBaud = None
-                def __init__(self):#, **kwargs):
+                def __init__(self, **kwargs):
-                    Operation.__init__(self)#, **kwargs)
+                    Operation.__init__(self, **kwargs)
                     self.times = None
                     self.osamp = None
                 #         self.__setValues = False
-                #    self.isConfig = False
+                    self.isConfig = False
                     self.setupReq = False
                 def setup(self, code, osamp, dataOut):
                 isConfig = False
                 buffer = None
-                def __init__(self):#, **kwargs):
+                def __init__(self, **kwargs):
-                    Operation.__init__(self)#, **kwargs)
+                    Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                 def reset(self):
                 # Tamanho total de los perfiles
                 nProfiles = None
-                def __init__(self):#, **kwargs):
+                def __init__(self, **kwargs):
-                    Operation.__init__(self)#, **kwargs)
+                    Operation.__init__(self, **kwargs)
                     self.profileIndex = 0
                 def incProfileIndex(self):
             class Reshaper(Operation):
-                def __init__(self):#, **kwargs):
+                def __init__(self, **kwargs):
-                    Operation.__init__(self)#, **kwargs)
+                    Operation.__init__(self, **kwargs)
                     self.__buffer = None
                     self.__nitems = 0
             class SplitProfiles(Operation):
-                def __init__(self):#, **kwargs):
+                def __init__(self, **kwargs):
-                    Operation.__init__(self)#, **kwargs)
+                    Operation.__init__(self, **kwargs)
                 def run(self, dataOut, n):
                     return dataOut
             class CombineProfiles(Operation):
-                def __init__(self):#, **kwargs):
+                def __init__(self, **kwargs):
-                    Operation.__init__(self)#, **kwargs)
+                    Operation.__init__(self, **kwargs)
                     self.__remData = None
                     self.__profileIndex = 0

General Comments 0

Write
Preview

You need to be logged in to leave comments. Login now

No TODOs yet

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages